{ "scope": "Exploratory existing splits; test has previously informed development", "calibration": { "global": 1.2589254117941675, "by_question": { "150440942f8fd3e04c6ffcb60c0215ffb1265e7d27e071f3ac453fd9817a85d4": 1.7378008287493754, "2a95b662ebe5bb2aa612f55ab9490ec200c83aec6e0ce1e9c439d769cb39a93c": 0.1, "330b97e5d96bf0d453555d62465367c41f1c5eb57ca53a1f3e8015c8ebe8548a": 0.1, "59786fb640773138e83576a42b1baff87c3a12c6924070c867a17f7541f8fd50": 1.0 }, "minimum_examples": 50, "unknown_question_policy": "global temperature; calibration not established" }, "splits": { "validation": { "uncalibrated": { "overall": { "count": 800, "accuracy": 0.93125, "macro_f1": 0.9441014135596143, "nll": 0.1488269731871437, "brier": 0.0975886054619946, "ece_10_bins": 0.017362924292683653, "oracle_expected_nll": 0.14322728173785357, "oracle_distribution_mse": 0.012396862257624472, "reliability": [ { "lower": 0.0, "upper": 0.1, "count": 0, "confidence": null, "accuracy": null }, { "lower": 0.1, "upper": 0.2, "count": 0, "confidence": null, "accuracy": null }, { "lower": 0.2, "upper": 0.3, "count": 0, "confidence": null, "accuracy": null }, { "lower": 0.3, "upper": 0.4, "count": 0, "confidence": null, "accuracy": null }, { "lower": 0.4, "upper": 0.5, "count": 0, "confidence": null, "accuracy": null }, { "lower": 0.5, "upper": 0.6, "count": 14, "confidence": 0.5606423233236585, "accuracy": 0.42857142857142855 }, { "lower": 0.6, "upper": 0.7, "count": 38, "confidence": 0.6494223318601909, "accuracy": 0.7105263157894737 }, { "lower": 0.7, "upper": 0.8, "count": 48, "confidence": 0.7586801064511141, "accuracy": 0.6875 }, { "lower": 0.8, "upper": 0.9, "count": 76, "confidence": 0.8506408341621098, "accuracy": 0.7763157894736842 }, { "lower": 0.9, "upper": 1.0, "count": 624, "confidence": 0.9925415913263956, "accuracy": 0.9935897435897436 } ], "risk_coverage": [ { "coverage": 0.25, "count": 200, "error_rate": 0.0 }, { "coverage": 0.5, "count": 400, "error_rate": 0.0 }, { "coverage": 0.75, "count": 600, "error_rate": 0.0033333333333332993 }, { "coverage": 1.0, "count": 800, "error_rate": 0.06874999999999998 } ] }, "by_task": { "department": { "count": 200, "accuracy": 0.995, "macro_f1": 0.9948307988219031, "nll": 0.02237316982799771, "brier": 0.010520559354480222, "ece_10_bins": 0.009969844520092013, "oracle_expected_nll": 0.02237316982799771, "oracle_distribution_mse": 0.010520559354480222, "reliability": [ { "lower": 0.0, "upper": 0.1, "count": 0, "confidence": null, "accuracy": null }, { "lower": 0.1, "upper": 0.2, "count": 0, "confidence": null, "accuracy": null }, { "lower": 0.2, "upper": 0.3, "count": 0, "confidence": null, "accuracy": null }, { "lower": 0.3, "upper": 0.4, "count": 0, "confidence": null, "accuracy": null }, { "lower": 0.4, "upper": 0.5, "count": 0, "confidence": null, "accuracy": null }, { "lower": 0.5, "upper": 0.6, "count": 0, "confidence": null, "accuracy": null }, { "lower": 0.6, "upper": 0.7, "count": 1, "confidence": 0.6584500074386597, "accuracy": 1.0 }, { "lower": 0.7, "upper": 0.8, "count": 1, "confidence": 0.7848847508430481, "accuracy": 1.0 }, { "lower": 0.8, "upper": 0.9, "count": 5, "confidence": 0.8563984274864197, "accuracy": 0.8 }, { "lower": 0.9, "upper": 1.0, "count": 193, "confidence": 0.9940139299229637, "accuracy": 1.0 } ], "risk_coverage": [ { "coverage": 0.25, "count": 50, "error_rate": 0.0 }, { "coverage": 0.5, "count": 100, "error_rate": 0.0 }, { "coverage": 0.75, "count": 150, "error_rate": 0.0 }, { "coverage": 1.0, "count": 200, "error_rate": 0.0050000000000000044 } ] }, "escalation": { "count": 200, "accuracy": 0.73, "macro_f1": 0.7282608695652174, "nll": 0.5679232407765421, "brier": 0.3794293531759783, "ece_10_bins": 0.07154568254947667, "oracle_expected_nll": 0.5455244749793817, "oracle_distribution_mse": 0.0386623803584977, "reliability": [ { "lower": 0.0, "upper": 0.1, "count": 0, "confidence": null, "accuracy": null }, { "lower": 0.1, "upper": 0.2, "count": 0, "confidence": null, "accuracy": null }, { "lower": 0.2, "upper": 0.3, "count": 0, "confidence": null, "accuracy": null }, { "lower": 0.3, "upper": 0.4, "count": 0, "confidence": null, "accuracy": null }, { "lower": 0.4, "upper": 0.5, "count": 0, "confidence": null, "accuracy": null }, { "lower": 0.5, "upper": 0.6, "count": 14, "confidence": 0.5606423233236585, "accuracy": 0.42857142857142855 }, { "lower": 0.6, "upper": 0.7, "count": 37, "confidence": 0.6491783406283405, "accuracy": 0.7027027027027027 }, { "lower": 0.7, "upper": 0.8, "count": 47, "confidence": 0.7581225608257537, "accuracy": 0.6808510638297872 }, { "lower": 0.8, "upper": 0.9, "count": 71, "confidence": 0.8502353698434965, "accuracy": 0.7746478873239436 }, { "lower": 0.9, "upper": 1.0, "count": 31, "confidence": 0.9187506764165817, "accuracy": 0.8709677419354839 } ], "risk_coverage": [ { "coverage": 0.25, "count": 50, "error_rate": 0.14 }, { "coverage": 0.5, "count": 100, "error_rate": 0.18999999999999995 }, { "coverage": 0.75, "count": 150, "error_rate": 0.23333333333333328 }, { "coverage": 1.0, "count": 200, "error_rate": 0.27 } ] }, "refund": { "count": 200, "accuracy": 1.0, "macro_f1": 1.0, "nll": 9.63352862168806e-05, "brier": 1.9762492451412852e-08, "ece_10_bins": 9.633034467693147e-05, "oracle_expected_nll": 9.63352862168806e-05, "oracle_distribution_mse": 1.9762492451412852e-08, "reliability": [ { "lower": 0.0, "upper": 0.1, "count": 0, "confidence": null, "accuracy": null }, { "lower": 0.1, "upper": 0.2, "count": 0, "confidence": null, "accuracy": null }, { "lower": 0.2, "upper": 0.3, "count": 0, "confidence": null, "accuracy": null }, { "lower": 0.3, "upper": 0.4, "count": 0, "confidence": null, "accuracy": null }, { "lower": 0.4, "upper": 0.5, "count": 0, "confidence": null, "accuracy": null }, { "lower": 0.5, "upper": 0.6, "count": 0, "confidence": null, "accuracy": null }, { "lower": 0.6, "upper": 0.7, "count": 0, "confidence": null, "accuracy": null }, { "lower": 0.7, "upper": 0.8, "count": 0, "confidence": null, "accuracy": null }, { "lower": 0.8, "upper": 0.9, "count": 0, "confidence": null, "accuracy": null }, { "lower": 0.9, "upper": 1.0, "count": 200, "confidence": 0.9999036696553231, "accuracy": 1.0 } ], "risk_coverage": [ { "coverage": 0.25, "count": 50, "error_rate": 0.0 }, { "coverage": 0.5, "count": 100, "error_rate": 0.0 }, { "coverage": 0.75, "count": 150, "error_rate": 0.0 }, { "coverage": 1.0, "count": 200, "error_rate": 0.0 } ] }, "severity": { "count": 200, "accuracy": 1.0, "macro_f1": 1.0, "nll": 0.004915146857817981, "brier": 0.0004044895550275171, "ece_10_bins": 0.00480370193719859, "oracle_expected_nll": 0.004915146857817981, "oracle_distribution_mse": 0.0004044895550275171, "reliability": [ { "lower": 0.0, "upper": 0.1, "count": 0, "confidence": null, "accuracy": null }, { "lower": 0.1, "upper": 0.2, "count": 0, "confidence": null, "accuracy": null }, { "lower": 0.2, "upper": 0.3, "count": 0, "confidence": null, "accuracy": null }, { "lower": 0.3, "upper": 0.4, "count": 0, "confidence": null, "accuracy": null }, { "lower": 0.4, "upper": 0.5, "count": 0, "confidence": null, "accuracy": null }, { "lower": 0.5, "upper": 0.6, "count": 0, "confidence": null, "accuracy": null }, { "lower": 0.6, "upper": 0.7, "count": 0, "confidence": null, "accuracy": null }, { "lower": 0.7, "upper": 0.8, "count": 0, "confidence": null, "accuracy": null }, { "lower": 0.8, "upper": 0.9, "count": 0, "confidence": null, "accuracy": null }, { "lower": 0.9, "upper": 1.0, "count": 200, "confidence": 0.9951962980628014, "accuracy": 1.0 } ], "risk_coverage": [ { "coverage": 0.25, "count": 50, "error_rate": 0.0 }, { "coverage": 0.5, "count": 100, "error_rate": 0.0 }, { "coverage": 0.75, "count": 150, "error_rate": 0.0 }, { "coverage": 1.0, "count": 200, "error_rate": 0.0 } ] } } }, "global": { "overall": { "count": 800, "accuracy": 0.93125, "macro_f1": 0.9441014135596143, "nll": 0.1497719984221153, "brier": 0.09681936358318226, "ece_10_bins": 0.00701747298240663, "oracle_expected_nll": 0.14532400513188456, "oracle_distribution_mse": 0.01255885871902599, "reliability": [ { "lower": 0.0, "upper": 0.1, "count": 0, "confidence": null, "accuracy": null }, { "lower": 0.1, "upper": 0.2, "count": 0, "confidence": null, "accuracy": null }, { "lower": 0.2, "upper": 0.3, "count": 0, "confidence": null, "accuracy": null }, { "lower": 0.3, "upper": 0.4, "count": 0, "confidence": null, "accuracy": null }, { "lower": 0.4, "upper": 0.5, "count": 0, "confidence": null, "accuracy": null }, { "lower": 0.5, "upper": 0.6, "count": 25, "confidence": 0.565165548324585, "accuracy": 0.52 }, { "lower": 0.6, "upper": 0.7, "count": 42, "confidence": 0.6500179114795867, "accuracy": 0.6666666666666666 }, { "lower": 0.7, "upper": 0.8, "count": 70, "confidence": 0.7533530805792127, "accuracy": 0.7571428571428571 }, { "lower": 0.8, "upper": 0.9, "count": 80, "confidence": 0.8482251234352589, "accuracy": 0.85 }, { "lower": 0.9, "upper": 1.0, "count": 583, "confidence": 0.9942052873493672, "accuracy": 1.0 } ], "risk_coverage": [ { "coverage": 0.25, "count": 200, "error_rate": 0.0 }, { "coverage": 0.5, "count": 400, "error_rate": 0.0 }, { "coverage": 0.75, "count": 600, "error_rate": 0.0033333333333332993 }, { "coverage": 1.0, "count": 800, "error_rate": 0.06874999999999998 } ] }, "by_task": { "department": { "count": 200, "accuracy": 0.995, "macro_f1": 0.9948307988219031, "nll": 0.029297618117778645, "brier": 0.011847873858568251, "ece_10_bins": 0.0152635979652405, "oracle_expected_nll": 0.029297618117778645, "oracle_distribution_mse": 0.011847873858568251, "reliability": [ { "lower": 0.0, "upper": 0.1, "count": 0, "confidence": null, "accuracy": null }, { "lower": 0.1, "upper": 0.2, "count": 0, "confidence": null, "accuracy": null }, { "lower": 0.2, "upper": 0.3, "count": 0, "confidence": null, "accuracy": null }, { "lower": 0.3, "upper": 0.4, "count": 0, "confidence": null, "accuracy": null }, { "lower": 0.4, "upper": 0.5, "count": 0, "confidence": null, "accuracy": null }, { "lower": 0.5, "upper": 0.6, "count": 0, "confidence": null, "accuracy": null }, { "lower": 0.6, "upper": 0.7, "count": 1, "confidence": 0.6272245645523071, "accuracy": 1.0 }, { "lower": 0.7, "upper": 0.8, "count": 2, "confidence": 0.7533551752567291, "accuracy": 1.0 }, { "lower": 0.8, "upper": 0.9, "count": 9, "confidence": 0.839881075753106, "accuracy": 0.8888888888888888 }, { "lower": 0.9, "upper": 1.0, "count": 188, "confidence": 0.9907149777133414, "accuracy": 1.0 } ], "risk_coverage": [ { "coverage": 0.25, "count": 50, "error_rate": 0.0 }, { "coverage": 0.5, "count": 100, "error_rate": 0.0 }, { "coverage": 0.75, "count": 150, "error_rate": 0.0 }, { "coverage": 1.0, "count": 200, "error_rate": 0.0050000000000000044 } ] }, "escalation": { "count": 200, "accuracy": 0.73, "macro_f1": 0.7282608695652174, "nll": 0.558092843042443, "brier": 0.37415878347224946, "ece_10_bins": 0.014070594906806932, "oracle_expected_nll": 0.54030086988152, "oracle_distribution_mse": 0.03711676401562454, "reliability": [ { "lower": 0.0, "upper": 0.1, "count": 0, "confidence": null, "accuracy": null }, { "lower": 0.1, "upper": 0.2, "count": 0, "confidence": null, "accuracy": null }, { "lower": 0.2, "upper": 0.3, "count": 0, "confidence": null, "accuracy": null }, { "lower": 0.3, "upper": 0.4, "count": 0, "confidence": null, "accuracy": null }, { "lower": 0.4, "upper": 0.5, "count": 0, "confidence": null, "accuracy": null }, { "lower": 0.5, "upper": 0.6, "count": 25, "confidence": 0.565165548324585, "accuracy": 0.52 }, { "lower": 0.6, "upper": 0.7, "count": 41, "confidence": 0.6505738467704959, "accuracy": 0.6585365853658537 }, { "lower": 0.7, "upper": 0.8, "count": 68, "confidence": 0.7533530189710504, "accuracy": 0.75 }, { "lower": 0.8, "upper": 0.9, "count": 65, "confidence": 0.8466363714291499, "accuracy": 0.8307692307692308 }, { "lower": 0.9, "upper": 1.0, "count": 1, "confidence": 0.9008614420890808, "accuracy": 1.0 } ], "risk_coverage": [ { "coverage": 0.25, "count": 50, "error_rate": 0.14 }, { "coverage": 0.5, "count": 100, "error_rate": 0.18999999999999995 }, { "coverage": 0.75, "count": 150, "error_rate": 0.23333333333333328 }, { "coverage": 1.0, "count": 200, "error_rate": 0.27 } ] }, "refund": { "count": 200, "accuracy": 1.0, "macro_f1": 1.0, "nll": 0.0006419544265103104, "brier": 8.562553287209336e-07, "ece_10_bins": 0.0006417402625084057, "oracle_expected_nll": 0.0006419544265103104, "oracle_distribution_mse": 8.562553287209336e-07, "reliability": [ { "lower": 0.0, "upper": 0.1, "count": 0, "confidence": null, "accuracy": null }, { "lower": 0.1, "upper": 0.2, "count": 0, "confidence": null, "accuracy": null }, { "lower": 0.2, "upper": 0.3, "count": 0, "confidence": null, "accuracy": null }, { "lower": 0.3, "upper": 0.4, "count": 0, "confidence": null, "accuracy": null }, { "lower": 0.4, "upper": 0.5, "count": 0, "confidence": null, "accuracy": null }, { "lower": 0.5, "upper": 0.6, "count": 0, "confidence": null, "accuracy": null }, { "lower": 0.6, "upper": 0.7, "count": 0, "confidence": null, "accuracy": null }, { "lower": 0.7, "upper": 0.8, "count": 0, "confidence": null, "accuracy": null }, { "lower": 0.8, "upper": 0.9, "count": 0, "confidence": null, "accuracy": null }, { "lower": 0.9, "upper": 1.0, "count": 200, "confidence": 0.9993582597374916, "accuracy": 1.0 } ], "risk_coverage": [ { "coverage": 0.25, "count": 50, "error_rate": 0.0 }, { "coverage": 0.5, "count": 100, "error_rate": 0.0 }, { "coverage": 0.75, "count": 150, "error_rate": 0.0 }, { "coverage": 1.0, "count": 200, "error_rate": 0.0 } ] }, "severity": { "count": 200, "accuracy": 1.0, "macro_f1": 1.0, "nll": 0.01105557810172927, "brier": 0.0012699407465824586, "ece_10_bins": 0.010687653124332395, "oracle_expected_nll": 0.01105557810172927, "oracle_distribution_mse": 0.0012699407465824586, "reliability": [ { "lower": 0.0, "upper": 0.1, "count": 0, "confidence": null, "accuracy": null }, { "lower": 0.1, "upper": 0.2, "count": 0, "confidence": null, "accuracy": null }, { "lower": 0.2, "upper": 0.3, "count": 0, "confidence": null, "accuracy": null }, { "lower": 0.3, "upper": 0.4, "count": 0, "confidence": null, "accuracy": null }, { "lower": 0.4, "upper": 0.5, "count": 0, "confidence": null, "accuracy": null }, { "lower": 0.5, "upper": 0.6, "count": 0, "confidence": null, "accuracy": null }, { "lower": 0.6, "upper": 0.7, "count": 0, "confidence": null, "accuracy": null }, { "lower": 0.7, "upper": 0.8, "count": 0, "confidence": null, "accuracy": null }, { "lower": 0.8, "upper": 0.9, "count": 6, "confidence": 0.8779526750246683, "accuracy": 1.0 }, { "lower": 0.9, "upper": 1.0, "count": 194, "confidence": 0.9927564604380696, "accuracy": 1.0 } ], "risk_coverage": [ { "coverage": 0.25, "count": 50, "error_rate": 0.0 }, { "coverage": 0.5, "count": 100, "error_rate": 0.0 }, { "coverage": 0.75, "count": 150, "error_rate": 0.0 }, { "coverage": 1.0, "count": 200, "error_rate": 0.0 } ] } } }, "per_question": { "overall": { "count": 800, "accuracy": 0.93125, "macro_f1": 0.9441014135596143, "nll": 0.1545261240547399, "brier": 0.09900029128593654, "ece_10_bins": 0.015734072476625435, "oracle_expected_nll": 0.14892643260544974, "oracle_distribution_mse": 0.013808548081566407, "reliability": [ { "lower": 0.0, "upper": 0.1, "count": 0, "confidence": null, "accuracy": null }, { "lower": 0.1, "upper": 0.2, "count": 0, "confidence": null, "accuracy": null }, { "lower": 0.2, "upper": 0.3, "count": 0, "confidence": null, "accuracy": null }, { "lower": 0.3, "upper": 0.4, "count": 0, "confidence": null, "accuracy": null }, { "lower": 0.4, "upper": 0.5, "count": 0, "confidence": null, "accuracy": null }, { "lower": 0.5, "upper": 0.6, "count": 15, "confidence": 0.5626973231633504, "accuracy": 0.4666666666666667 }, { "lower": 0.6, "upper": 0.7, "count": 38, "confidence": 0.6498527918991289, "accuracy": 0.7105263157894737 }, { "lower": 0.7, "upper": 0.8, "count": 56, "confidence": 0.7574487828782627, "accuracy": 0.7142857142857143 }, { "lower": 0.8, "upper": 0.9, "count": 81, "confidence": 0.8516954691321762, "accuracy": 0.8024691358024691 }, { "lower": 0.9, "upper": 1.0, "count": 610, "confidence": 0.9894479682211016, "accuracy": 0.9934426229508196 } ], "risk_coverage": [ { "coverage": 0.25, "count": 200, "error_rate": 0.0 }, { "coverage": 0.5, "count": 400, "error_rate": 0.0 }, { "coverage": 0.75, "count": 600, "error_rate": 0.00666666666666671 }, { "coverage": 1.0, "count": 800, "error_rate": 0.06874999999999998 } ] }, "by_task": { "department": { "count": 200, "accuracy": 0.995, "macro_f1": 0.9948307988219031, "nll": 0.05018125544241732, "brier": 0.016571811967767927, "ece_10_bins": 0.03622871160507202, "oracle_expected_nll": 0.05018125544241732, "oracle_distribution_mse": 0.016571811967767927, "reliability": [ { "lower": 0.0, "upper": 0.1, "count": 0, "confidence": null, "accuracy": null }, { "lower": 0.1, "upper": 0.2, "count": 0, "confidence": null, "accuracy": null }, { "lower": 0.2, "upper": 0.3, "count": 0, "confidence": null, "accuracy": null }, { "lower": 0.3, "upper": 0.4, "count": 0, "confidence": null, "accuracy": null }, { "lower": 0.4, "upper": 0.5, "count": 0, "confidence": null, "accuracy": null }, { "lower": 0.5, "upper": 0.6, "count": 1, "confidence": 0.5914673209190369, "accuracy": 1.0 }, { "lower": 0.6, "upper": 0.7, "count": 1, "confidence": 0.6748074889183044, "accuracy": 1.0 }, { "lower": 0.7, "upper": 0.8, "count": 9, "confidence": 0.7539301647080315, "accuracy": 0.8888888888888888 }, { "lower": 0.8, "upper": 0.9, "count": 10, "confidence": 0.8620621740818024, "accuracy": 1.0 }, { "lower": 0.9, "upper": 1.0, "count": 179, "confidence": 0.9781116739997651, "accuracy": 1.0 } ], "risk_coverage": [ { "coverage": 0.25, "count": 50, "error_rate": 0.0 }, { "coverage": 0.5, "count": 100, "error_rate": 0.0 }, { "coverage": 0.75, "count": 150, "error_rate": 0.0 }, { "coverage": 1.0, "count": 200, "error_rate": 0.0050000000000000044 } ] }, "escalation": { "count": 200, "accuracy": 0.73, "macro_f1": 0.7282608695652174, "nll": 0.5679232407765421, "brier": 0.3794293531759783, "ece_10_bins": 0.07154568254947667, "oracle_expected_nll": 0.5455244749793817, "oracle_distribution_mse": 0.0386623803584977, "reliability": [ { "lower": 0.0, "upper": 0.1, "count": 0, "confidence": null, "accuracy": null }, { "lower": 0.1, "upper": 0.2, "count": 0, "confidence": null, "accuracy": null }, { "lower": 0.2, "upper": 0.3, "count": 0, "confidence": null, "accuracy": null }, { "lower": 0.3, "upper": 0.4, "count": 0, "confidence": null, "accuracy": null }, { "lower": 0.4, "upper": 0.5, "count": 0, "confidence": null, "accuracy": null }, { "lower": 0.5, "upper": 0.6, "count": 14, "confidence": 0.5606423233236585, "accuracy": 0.42857142857142855 }, { "lower": 0.6, "upper": 0.7, "count": 37, "confidence": 0.6491783406283405, "accuracy": 0.7027027027027027 }, { "lower": 0.7, "upper": 0.8, "count": 47, "confidence": 0.7581225608257537, "accuracy": 0.6808510638297872 }, { "lower": 0.8, "upper": 0.9, "count": 71, "confidence": 0.8502353698434965, "accuracy": 0.7746478873239436 }, { "lower": 0.9, "upper": 1.0, "count": 31, "confidence": 0.9187506764165817, "accuracy": 0.8709677419354839 } ], "risk_coverage": [ { "coverage": 0.25, "count": 50, "error_rate": 0.14 }, { "coverage": 0.5, "count": 100, "error_rate": 0.18999999999999995 }, { "coverage": 0.75, "count": 150, "error_rate": 0.23333333333333328 }, { "coverage": 1.0, "count": 200, "error_rate": 0.27 } ] }, "refund": { "count": 200, "accuracy": 1.0, "macro_f1": 1.0, "nll": 0.0, "brier": 4.3804362436076516e-76, "ece_10_bins": 0.0, "oracle_expected_nll": 0.0, "oracle_distribution_mse": 4.3804362436076516e-76, "reliability": [ { "lower": 0.0, "upper": 0.1, "count": 0, "confidence": null, "accuracy": null }, { "lower": 0.1, "upper": 0.2, "count": 0, "confidence": null, "accuracy": null }, { "lower": 0.2, "upper": 0.3, "count": 0, "confidence": null, "accuracy": null }, { "lower": 0.3, "upper": 0.4, "count": 0, "confidence": null, "accuracy": null }, { "lower": 0.4, "upper": 0.5, "count": 0, "confidence": null, "accuracy": null }, { "lower": 0.5, "upper": 0.6, "count": 0, "confidence": null, "accuracy": null }, { "lower": 0.6, "upper": 0.7, "count": 0, "confidence": null, "accuracy": null }, { "lower": 0.7, "upper": 0.8, "count": 0, "confidence": null, "accuracy": null }, { "lower": 0.8, "upper": 0.9, "count": 0, "confidence": null, "accuracy": null }, { "lower": 0.9, "upper": 1.0, "count": 200, "confidence": 1.0, "accuracy": 1.0 } ], "risk_coverage": [ { "coverage": 0.25, "count": 50, "error_rate": 0.0 }, { "coverage": 0.5, "count": 100, "error_rate": 0.0 }, { "coverage": 0.75, "count": 150, "error_rate": 0.0 }, { "coverage": 1.0, "count": 200, "error_rate": 0.0 } ] }, "severity": { "count": 200, "accuracy": 1.0, "macro_f1": 1.0, "nll": 0.0, "brier": 7.613875406184365e-24, "ece_10_bins": 0.0, "oracle_expected_nll": 0.0, "oracle_distribution_mse": 7.613875406184365e-24, "reliability": [ { "lower": 0.0, "upper": 0.1, "count": 0, "confidence": null, "accuracy": null }, { "lower": 0.1, "upper": 0.2, "count": 0, "confidence": null, "accuracy": null }, { "lower": 0.2, "upper": 0.3, "count": 0, "confidence": null, "accuracy": null }, { "lower": 0.3, "upper": 0.4, "count": 0, "confidence": null, "accuracy": null }, { "lower": 0.4, "upper": 0.5, "count": 0, "confidence": null, "accuracy": null }, { "lower": 0.5, "upper": 0.6, "count": 0, "confidence": null, "accuracy": null }, { "lower": 0.6, "upper": 0.7, "count": 0, "confidence": null, "accuracy": null }, { "lower": 0.7, "upper": 0.8, "count": 0, "confidence": null, "accuracy": null }, { "lower": 0.8, "upper": 0.9, "count": 0, "confidence": null, "accuracy": null }, { "lower": 0.9, "upper": 1.0, "count": 200, "confidence": 1.0, "accuracy": 1.0 } ], "risk_coverage": [ { "coverage": 0.25, "count": 50, "error_rate": 0.0 }, { "coverage": 0.5, "count": 100, "error_rate": 0.0 }, { "coverage": 0.75, "count": 150, "error_rate": 0.0 }, { "coverage": 1.0, "count": 200, "error_rate": 0.0 } ] } } } }, "test": { "uncalibrated": { "overall": { "count": 800, "accuracy": 0.925, "macro_f1": 0.9392552790275188, "nll": 0.15805856801203613, "brier": 0.10519841661327942, "ece_10_bins": 0.025654533132910683, "oracle_expected_nll": 0.138174928971103, "oracle_distribution_mse": 0.01090862890356645, "reliability": [ { "lower": 0.0, "upper": 0.1, "count": 0, "confidence": null, "accuracy": null }, { "lower": 0.1, "upper": 0.2, "count": 0, "confidence": null, "accuracy": null }, { "lower": 0.2, "upper": 0.3, "count": 0, "confidence": null, "accuracy": null }, { "lower": 0.3, "upper": 0.4, "count": 0, "confidence": null, "accuracy": null }, { "lower": 0.4, "upper": 0.5, "count": 0, "confidence": null, "accuracy": null }, { "lower": 0.5, "upper": 0.6, "count": 17, "confidence": 0.5612775718464571, "accuracy": 0.23529411764705882 }, { "lower": 0.6, "upper": 0.7, "count": 16, "confidence": 0.6538418717682362, "accuracy": 0.75 }, { "lower": 0.7, "upper": 0.8, "count": 64, "confidence": 0.7603475973010063, "accuracy": 0.765625 }, { "lower": 0.8, "upper": 0.9, "count": 75, "confidence": 0.8518771576881409, "accuracy": 0.68 }, { "lower": 0.9, "upper": 1.0, "count": 628, "confidence": 0.9939726705953573, "accuracy": 0.9936305732484076 } ], "risk_coverage": [ { "coverage": 0.25, "count": 200, "error_rate": 0.0 }, { "coverage": 0.5, "count": 400, "error_rate": 0.0 }, { "coverage": 0.75, "count": 600, "error_rate": 0.0 }, { "coverage": 1.0, "count": 800, "error_rate": 0.07499999999999996 } ] }, "by_task": { "department": { "count": 200, "accuracy": 0.995, "macro_f1": 0.9944254755766148, "nll": 0.013614306728828655, "brier": 0.006972022766819956, "ece_10_bins": 0.00483260512351995, "oracle_expected_nll": 0.013614306728828655, "oracle_distribution_mse": 0.006972022766819956, "reliability": [ { "lower": 0.0, "upper": 0.1, "count": 0, "confidence": null, "accuracy": null }, { "lower": 0.1, "upper": 0.2, "count": 0, "confidence": null, "accuracy": null }, { "lower": 0.2, "upper": 0.3, "count": 0, "confidence": null, "accuracy": null }, { "lower": 0.3, "upper": 0.4, "count": 0, "confidence": null, "accuracy": null }, { "lower": 0.4, "upper": 0.5, "count": 0, "confidence": null, "accuracy": null }, { "lower": 0.5, "upper": 0.6, "count": 0, "confidence": null, "accuracy": null }, { "lower": 0.6, "upper": 0.7, "count": 3, "confidence": 0.6343480944633484, "accuracy": 0.6666666666666666 }, { "lower": 0.7, "upper": 0.8, "count": 0, "confidence": null, "accuracy": null }, { "lower": 0.8, "upper": 0.9, "count": 3, "confidence": 0.8595178723335266, "accuracy": 1.0 }, { "lower": 0.9, "upper": 1.0, "count": 194, "confidence": 0.9976901086335329, "accuracy": 1.0 } ], "risk_coverage": [ { "coverage": 0.25, "count": 50, "error_rate": 0.0 }, { "coverage": 0.5, "count": 100, "error_rate": 0.0 }, { "coverage": 0.75, "count": 150, "error_rate": 0.0 }, { "coverage": 1.0, "count": 200, "error_rate": 0.0050000000000000044 } ] }, "escalation": { "count": 200, "accuracy": 0.705, "macro_f1": 0.7046381817726715, "nll": 0.6145763126252539, "brier": 0.4129853920524647, "ece_10_bins": 0.11028719127178192, "oracle_expected_nll": 0.5350417564615215, "oracle_distribution_mse": 0.03582624121361282, "reliability": [ { "lower": 0.0, "upper": 0.1, "count": 0, "confidence": null, "accuracy": null }, { "lower": 0.1, "upper": 0.2, "count": 0, "confidence": null, "accuracy": null }, { "lower": 0.2, "upper": 0.3, "count": 0, "confidence": null, "accuracy": null }, { "lower": 0.3, "upper": 0.4, "count": 0, "confidence": null, "accuracy": null }, { "lower": 0.4, "upper": 0.5, "count": 0, "confidence": null, "accuracy": null }, { "lower": 0.5, "upper": 0.6, "count": 17, "confidence": 0.5612775718464571, "accuracy": 0.23529411764705882 }, { "lower": 0.6, "upper": 0.7, "count": 13, "confidence": 0.6583404357616718, "accuracy": 0.7692307692307693 }, { "lower": 0.7, "upper": 0.8, "count": 64, "confidence": 0.7603475973010063, "accuracy": 0.765625 }, { "lower": 0.8, "upper": 0.9, "count": 69, "confidence": 0.8522450759790946, "accuracy": 0.6521739130434783 }, { "lower": 0.9, "upper": 1.0, "count": 37, "confidence": 0.9170670589885196, "accuracy": 0.8918918918918919 } ], "risk_coverage": [ { "coverage": 0.25, "count": 50, "error_rate": 0.19999999999999996 }, { "coverage": 0.5, "count": 100, "error_rate": 0.26 }, { "coverage": 0.75, "count": 150, "error_rate": 0.2733333333333333 }, { "coverage": 1.0, "count": 200, "error_rate": 0.29500000000000004 } ] }, "refund": { "count": 200, "accuracy": 1.0, "macro_f1": 1.0, "nll": 9.923051486518732e-05, "brier": 2.306917757809561e-08, "ece_10_bins": 9.922474622725552e-05, "oracle_expected_nll": 9.923051486518732e-05, "oracle_distribution_mse": 2.306917757809561e-08, "reliability": [ { "lower": 0.0, "upper": 0.1, "count": 0, "confidence": null, "accuracy": null }, { "lower": 0.1, "upper": 0.2, "count": 0, "confidence": null, "accuracy": null }, { "lower": 0.2, "upper": 0.3, "count": 0, "confidence": null, "accuracy": null }, { "lower": 0.3, "upper": 0.4, "count": 0, "confidence": null, "accuracy": null }, { "lower": 0.4, "upper": 0.5, "count": 0, "confidence": null, "accuracy": null }, { "lower": 0.5, "upper": 0.6, "count": 0, "confidence": null, "accuracy": null }, { "lower": 0.6, "upper": 0.7, "count": 0, "confidence": null, "accuracy": null }, { "lower": 0.7, "upper": 0.8, "count": 0, "confidence": null, "accuracy": null }, { "lower": 0.8, "upper": 0.9, "count": 0, "confidence": null, "accuracy": null }, { "lower": 0.9, "upper": 1.0, "count": 200, "confidence": 0.9999007752537727, "accuracy": 1.0 } ], "risk_coverage": [ { "coverage": 0.25, "count": 50, "error_rate": 0.0 }, { "coverage": 0.5, "count": 100, "error_rate": 0.0 }, { "coverage": 0.75, "count": 150, "error_rate": 0.0 }, { "coverage": 1.0, "count": 200, "error_rate": 0.0 } ] }, "severity": { "count": 200, "accuracy": 1.0, "macro_f1": 1.0, "nll": 0.003944422179196725, "brier": 0.000836228564655437, "ece_10_bins": 0.00370678603649136, "oracle_expected_nll": 0.003944422179196725, "oracle_distribution_mse": 0.000836228564655437, "reliability": [ { "lower": 0.0, "upper": 0.1, "count": 0, "confidence": null, "accuracy": null }, { "lower": 0.1, "upper": 0.2, "count": 0, "confidence": null, "accuracy": null }, { "lower": 0.2, "upper": 0.3, "count": 0, "confidence": null, "accuracy": null }, { "lower": 0.3, "upper": 0.4, "count": 0, "confidence": null, "accuracy": null }, { "lower": 0.4, "upper": 0.5, "count": 0, "confidence": null, "accuracy": null }, { "lower": 0.5, "upper": 0.6, "count": 0, "confidence": null, "accuracy": null }, { "lower": 0.6, "upper": 0.7, "count": 0, "confidence": null, "accuracy": null }, { "lower": 0.7, "upper": 0.8, "count": 0, "confidence": null, "accuracy": null }, { "lower": 0.8, "upper": 0.9, "count": 3, "confidence": 0.8357743223508199, "accuracy": 1.0 }, { "lower": 0.9, "upper": 1.0, "count": 197, "confidence": 0.9987376640895902, "accuracy": 1.0 } ], "risk_coverage": [ { "coverage": 0.25, "count": 50, "error_rate": 0.0 }, { "coverage": 0.5, "count": 100, "error_rate": 0.0 }, { "coverage": 0.75, "count": 150, "error_rate": 0.0 }, { "coverage": 1.0, "count": 200, "error_rate": 0.0 } ] } } }, "global": { "overall": { "count": 800, "accuracy": 0.925, "macro_f1": 0.9392552790275188, "nll": 0.15451136706332724, "brier": 0.10279470684407294, "ece_10_bins": 0.02014885999262334, "oracle_expected_nll": 0.1387172312235814, "oracle_distribution_mse": 0.010962450171169045, "reliability": [ { "lower": 0.0, "upper": 0.1, "count": 0, "confidence": null, "accuracy": null }, { "lower": 0.1, "upper": 0.2, "count": 0, "confidence": null, "accuracy": null }, { "lower": 0.2, "upper": 0.3, "count": 0, "confidence": null, "accuracy": null }, { "lower": 0.3, "upper": 0.4, "count": 0, "confidence": null, "accuracy": null }, { "lower": 0.4, "upper": 0.5, "count": 0, "confidence": null, "accuracy": null }, { "lower": 0.5, "upper": 0.6, "count": 22, "confidence": 0.5578833384947344, "accuracy": 0.36363636363636365 }, { "lower": 0.6, "upper": 0.7, "count": 24, "confidence": 0.6626042500138283, "accuracy": 0.7916666666666666 }, { "lower": 0.7, "upper": 0.8, "count": 86, "confidence": 0.7429546443528907, "accuracy": 0.7441860465116279 }, { "lower": 0.8, "upper": 0.9, "count": 79, "confidence": 0.8482364740552781, "accuracy": 0.759493670886076 }, { "lower": 0.9, "upper": 1.0, "count": 589, "confidence": 0.9972299244286451, "accuracy": 1.0 } ], "risk_coverage": [ { "coverage": 0.25, "count": 200, "error_rate": 0.0 }, { "coverage": 0.5, "count": 400, "error_rate": 0.0 }, { "coverage": 0.75, "count": 600, "error_rate": 0.0 }, { "coverage": 1.0, "count": 800, "error_rate": 0.07499999999999996 } ] }, "by_task": { "department": { "count": 200, "accuracy": 0.995, "macro_f1": 0.9944254755766148, "nll": 0.017469275119756628, "brier": 0.008068493609724506, "ece_10_bins": 0.010246815681457555, "oracle_expected_nll": 0.017469275119756628, "oracle_distribution_mse": 0.008068493609724506, "reliability": [ { "lower": 0.0, "upper": 0.1, "count": 0, "confidence": null, "accuracy": null }, { "lower": 0.1, "upper": 0.2, "count": 0, "confidence": null, "accuracy": null }, { "lower": 0.2, "upper": 0.3, "count": 0, "confidence": null, "accuracy": null }, { "lower": 0.3, "upper": 0.4, "count": 0, "confidence": null, "accuracy": null }, { "lower": 0.4, "upper": 0.5, "count": 0, "confidence": null, "accuracy": null }, { "lower": 0.5, "upper": 0.6, "count": 2, "confidence": 0.5836484432220459, "accuracy": 0.5 }, { "lower": 0.6, "upper": 0.7, "count": 1, "confidence": 0.6414172649383545, "accuracy": 1.0 }, { "lower": 0.7, "upper": 0.8, "count": 1, "confidence": 0.7566628456115723, "accuracy": 1.0 }, { "lower": 0.8, "upper": 0.9, "count": 4, "confidence": 0.8481354713439941, "accuracy": 1.0 }, { "lower": 0.9, "upper": 1.0, "count": 192, "confidence": 0.9964964153865973, "accuracy": 1.0 } ], "risk_coverage": [ { "coverage": 0.25, "count": 50, "error_rate": 0.0 }, { "coverage": 0.5, "count": 100, "error_rate": 0.0 }, { "coverage": 0.75, "count": 150, "error_rate": 0.0 }, { "coverage": 1.0, "count": 200, "error_rate": 0.0050000000000000044 } ] }, "escalation": { "count": 200, "accuracy": 0.705, "macro_f1": 0.7046381817726715, "nll": 0.5920632934974356, "brier": 0.4015789899739018, "ece_10_bins": 0.07626515358686448, "oracle_expected_nll": 0.5288867501384523, "oracle_distribution_mse": 0.03424996328228628, "reliability": [ { "lower": 0.0, "upper": 0.1, "count": 0, "confidence": null, "accuracy": null }, { "lower": 0.1, "upper": 0.2, "count": 0, "confidence": null, "accuracy": null }, { "lower": 0.2, "upper": 0.3, "count": 0, "confidence": null, "accuracy": null }, { "lower": 0.3, "upper": 0.4, "count": 0, "confidence": null, "accuracy": null }, { "lower": 0.4, "upper": 0.5, "count": 0, "confidence": null, "accuracy": null }, { "lower": 0.5, "upper": 0.6, "count": 20, "confidence": 0.5553068280220032, "accuracy": 0.35 }, { "lower": 0.6, "upper": 0.7, "count": 23, "confidence": 0.6635254232779794, "accuracy": 0.782608695652174 }, { "lower": 0.7, "upper": 0.8, "count": 83, "confidence": 0.7421774620033172, "accuracy": 0.7349397590361446 }, { "lower": 0.8, "upper": 0.9, "count": 74, "confidence": 0.848746615487176, "accuracy": 0.7432432432432432 }, { "lower": 0.9, "upper": 1.0, "count": 0, "confidence": null, "accuracy": null } ], "risk_coverage": [ { "coverage": 0.25, "count": 50, "error_rate": 0.19999999999999996 }, { "coverage": 0.5, "count": 100, "error_rate": 0.26 }, { "coverage": 0.75, "count": 150, "error_rate": 0.2733333333333333 }, { "coverage": 1.0, "count": 200, "error_rate": 0.29500000000000004 } ] }, "refund": { "count": 200, "accuracy": 1.0, "macro_f1": 1.0, "nll": 0.0006544187157477001, "brier": 9.303064614080757e-07, "ece_10_bins": 0.0006541860103607133, "oracle_expected_nll": 0.0006544187157477001, "oracle_distribution_mse": 9.303064614080757e-07, "reliability": [ { "lower": 0.0, "upper": 0.1, "count": 0, "confidence": null, "accuracy": null }, { "lower": 0.1, "upper": 0.2, "count": 0, "confidence": null, "accuracy": null }, { "lower": 0.2, "upper": 0.3, "count": 0, "confidence": null, "accuracy": null }, { "lower": 0.3, "upper": 0.4, "count": 0, "confidence": null, "accuracy": null }, { "lower": 0.4, "upper": 0.5, "count": 0, "confidence": null, "accuracy": null }, { "lower": 0.5, "upper": 0.6, "count": 0, "confidence": null, "accuracy": null }, { "lower": 0.6, "upper": 0.7, "count": 0, "confidence": null, "accuracy": null }, { "lower": 0.7, "upper": 0.8, "count": 0, "confidence": null, "accuracy": null }, { "lower": 0.8, "upper": 0.9, "count": 0, "confidence": null, "accuracy": null }, { "lower": 0.9, "upper": 1.0, "count": 200, "confidence": 0.9993458139896393, "accuracy": 1.0 } ], "risk_coverage": [ { "coverage": 0.25, "count": 50, "error_rate": 0.0 }, { "coverage": 0.5, "count": 100, "error_rate": 0.0 }, { "coverage": 0.75, "count": 150, "error_rate": 0.0 }, { "coverage": 1.0, "count": 200, "error_rate": 0.0 } ] }, "severity": { "count": 200, "accuracy": 1.0, "macro_f1": 1.0, "nll": 0.007858480920368994, "brier": 0.0015304134862039825, "ece_10_bins": 0.007402259111404373, "oracle_expected_nll": 0.007858480920368994, "oracle_distribution_mse": 0.0015304134862039825, "reliability": [ { "lower": 0.0, "upper": 0.1, "count": 0, "confidence": null, "accuracy": null }, { "lower": 0.1, "upper": 0.2, "count": 0, "confidence": null, "accuracy": null }, { "lower": 0.2, "upper": 0.3, "count": 0, "confidence": null, "accuracy": null }, { "lower": 0.3, "upper": 0.4, "count": 0, "confidence": null, "accuracy": null }, { "lower": 0.4, "upper": 0.5, "count": 0, "confidence": null, "accuracy": null }, { "lower": 0.5, "upper": 0.6, "count": 0, "confidence": null, "accuracy": null }, { "lower": 0.6, "upper": 0.7, "count": 0, "confidence": null, "accuracy": null }, { "lower": 0.7, "upper": 0.8, "count": 2, "confidence": 0.7683536112308502, "accuracy": 1.0 }, { "lower": 0.8, "upper": 0.9, "count": 1, "confidence": 0.8108900189399719, "accuracy": 1.0 }, { "lower": 0.9, "upper": 1.0, "count": 197, "confidence": 0.9957967052604947, "accuracy": 1.0 } ], "risk_coverage": [ { "coverage": 0.25, "count": 50, "error_rate": 0.0 }, { "coverage": 0.5, "count": 100, "error_rate": 0.0 }, { "coverage": 0.75, "count": 150, "error_rate": 0.0 }, { "coverage": 1.0, "count": 200, "error_rate": 0.0 } ] } } }, "per_question": { "overall": { "count": 800, "accuracy": 0.925, "macro_f1": 0.9392552790275188, "nll": 0.16079165909777055, "brier": 0.10593323494171354, "ece_10_bins": 0.02878171861171729, "oracle_expected_nll": 0.14090802005683742, "oracle_distribution_mse": 0.01164344723200057, "reliability": [ { "lower": 0.0, "upper": 0.1, "count": 0, "confidence": null, "accuracy": null }, { "lower": 0.1, "upper": 0.2, "count": 0, "confidence": null, "accuracy": null }, { "lower": 0.2, "upper": 0.3, "count": 0, "confidence": null, "accuracy": null }, { "lower": 0.3, "upper": 0.4, "count": 0, "confidence": null, "accuracy": null }, { "lower": 0.4, "upper": 0.5, "count": 0, "confidence": null, "accuracy": null }, { "lower": 0.5, "upper": 0.6, "count": 19, "confidence": 0.5593467480257938, "accuracy": 0.2631578947368421 }, { "lower": 0.6, "upper": 0.7, "count": 15, "confidence": 0.6569308956464132, "accuracy": 0.8 }, { "lower": 0.7, "upper": 0.8, "count": 68, "confidence": 0.7610192018396714, "accuracy": 0.7794117647058824 }, { "lower": 0.8, "upper": 0.9, "count": 73, "confidence": 0.8525561878125961, "accuracy": 0.6712328767123288 }, { "lower": 0.9, "upper": 1.0, "count": 625, "confidence": 0.9923768733978271, "accuracy": 0.9936 } ], "risk_coverage": [ { "coverage": 0.25, "count": 200, "error_rate": 0.0 }, { "coverage": 0.5, "count": 400, "error_rate": 0.0 }, { "coverage": 0.75, "count": 600, "error_rate": 0.0016666666666667052 }, { "coverage": 1.0, "count": 800, "error_rate": 0.07499999999999996 } ] }, "by_task": { "department": { "count": 200, "accuracy": 0.995, "macro_f1": 0.9944254755766148, "nll": 0.02859032138164205, "brier": 0.010747547714388071, "ece_10_bins": 0.01983757793903353, "oracle_expected_nll": 0.02859032138164205, "oracle_distribution_mse": 0.010747547714388071, "reliability": [ { "lower": 0.0, "upper": 0.1, "count": 0, "confidence": null, "accuracy": null }, { "lower": 0.1, "upper": 0.2, "count": 0, "confidence": null, "accuracy": null }, { "lower": 0.2, "upper": 0.3, "count": 0, "confidence": null, "accuracy": null }, { "lower": 0.3, "upper": 0.4, "count": 0, "confidence": null, "accuracy": null }, { "lower": 0.4, "upper": 0.5, "count": 0, "confidence": null, "accuracy": null }, { "lower": 0.5, "upper": 0.6, "count": 2, "confidence": 0.5429347455501556, "accuracy": 0.5 }, { "lower": 0.6, "upper": 0.7, "count": 2, "confidence": 0.6477688848972321, "accuracy": 1.0 }, { "lower": 0.7, "upper": 0.8, "count": 4, "confidence": 0.771764874458313, "accuracy": 1.0 }, { "lower": 0.8, "upper": 0.9, "count": 4, "confidence": 0.8579228669404984, "accuracy": 1.0 }, { "lower": 0.9, "upper": 1.0, "count": 188, "confidence": 0.990979070042042, "accuracy": 1.0 } ], "risk_coverage": [ { "coverage": 0.25, "count": 50, "error_rate": 0.0 }, { "coverage": 0.5, "count": 100, "error_rate": 0.0 }, { "coverage": 0.75, "count": 150, "error_rate": 0.0 }, { "coverage": 1.0, "count": 200, "error_rate": 0.0050000000000000044 } ] }, "escalation": { "count": 200, "accuracy": 0.705, "macro_f1": 0.7046381817726715, "nll": 0.6145763126252539, "brier": 0.4129853920524647, "ece_10_bins": 0.11028719127178192, "oracle_expected_nll": 0.5350417564615215, "oracle_distribution_mse": 0.03582624121361282, "reliability": [ { "lower": 0.0, "upper": 0.1, "count": 0, "confidence": null, "accuracy": null }, { "lower": 0.1, "upper": 0.2, "count": 0, "confidence": null, "accuracy": null }, { "lower": 0.2, "upper": 0.3, "count": 0, "confidence": null, "accuracy": null }, { "lower": 0.3, "upper": 0.4, "count": 0, "confidence": null, "accuracy": null }, { "lower": 0.4, "upper": 0.5, "count": 0, "confidence": null, "accuracy": null }, { "lower": 0.5, "upper": 0.6, "count": 17, "confidence": 0.5612775718464571, "accuracy": 0.23529411764705882 }, { "lower": 0.6, "upper": 0.7, "count": 13, "confidence": 0.6583404357616718, "accuracy": 0.7692307692307693 }, { "lower": 0.7, "upper": 0.8, "count": 64, "confidence": 0.7603475973010063, "accuracy": 0.765625 }, { "lower": 0.8, "upper": 0.9, "count": 69, "confidence": 0.8522450759790946, "accuracy": 0.6521739130434783 }, { "lower": 0.9, "upper": 1.0, "count": 37, "confidence": 0.9170670589885196, "accuracy": 0.8918918918918919 } ], "risk_coverage": [ { "coverage": 0.25, "count": 50, "error_rate": 0.19999999999999996 }, { "coverage": 0.5, "count": 100, "error_rate": 0.26 }, { "coverage": 0.75, "count": 150, "error_rate": 0.2733333333333333 }, { "coverage": 1.0, "count": 200, "error_rate": 0.29500000000000004 } ] }, "refund": { "count": 200, "accuracy": 1.0, "macro_f1": 1.0, "nll": 0.0, "brier": 6.297547947730489e-70, "ece_10_bins": 0.0, "oracle_expected_nll": 0.0, "oracle_distribution_mse": 6.297547947730489e-70, "reliability": [ { "lower": 0.0, "upper": 0.1, "count": 0, "confidence": null, "accuracy": null }, { "lower": 0.1, "upper": 0.2, "count": 0, "confidence": null, "accuracy": null }, { "lower": 0.2, "upper": 0.3, "count": 0, "confidence": null, "accuracy": null }, { "lower": 0.3, "upper": 0.4, "count": 0, "confidence": null, "accuracy": null }, { "lower": 0.4, "upper": 0.5, "count": 0, "confidence": null, "accuracy": null }, { "lower": 0.5, "upper": 0.6, "count": 0, "confidence": null, "accuracy": null }, { "lower": 0.6, "upper": 0.7, "count": 0, "confidence": null, "accuracy": null }, { "lower": 0.7, "upper": 0.8, "count": 0, "confidence": null, "accuracy": null }, { "lower": 0.8, "upper": 0.9, "count": 0, "confidence": null, "accuracy": null }, { "lower": 0.9, "upper": 1.0, "count": 200, "confidence": 1.0, "accuracy": 1.0 } ], "risk_coverage": [ { "coverage": 0.25, "count": 50, "error_rate": 0.0 }, { "coverage": 0.5, "count": 100, "error_rate": 0.0 }, { "coverage": 0.75, "count": 150, "error_rate": 0.0 }, { "coverage": 1.0, "count": 200, "error_rate": 0.0 } ] }, "severity": { "count": 200, "accuracy": 1.0, "macro_f1": 1.0, "nll": 2.384186146287072e-09, "brier": 1.3848676160326649e-15, "ece_10_bins": 2.38418573772492e-09, "oracle_expected_nll": 2.384186146287072e-09, "oracle_distribution_mse": 1.3848676160326649e-15, "reliability": [ { "lower": 0.0, "upper": 0.1, "count": 0, "confidence": null, "accuracy": null }, { "lower": 0.1, "upper": 0.2, "count": 0, "confidence": null, "accuracy": null }, { "lower": 0.2, "upper": 0.3, "count": 0, "confidence": null, "accuracy": null }, { "lower": 0.3, "upper": 0.4, "count": 0, "confidence": null, "accuracy": null }, { "lower": 0.4, "upper": 0.5, "count": 0, "confidence": null, "accuracy": null }, { "lower": 0.5, "upper": 0.6, "count": 0, "confidence": null, "accuracy": null }, { "lower": 0.6, "upper": 0.7, "count": 0, "confidence": null, "accuracy": null }, { "lower": 0.7, "upper": 0.8, "count": 0, "confidence": null, "accuracy": null }, { "lower": 0.8, "upper": 0.9, "count": 0, "confidence": null, "accuracy": null }, { "lower": 0.9, "upper": 1.0, "count": 200, "confidence": 0.9999999976158143, "accuracy": 1.0 } ], "risk_coverage": [ { "coverage": 0.25, "count": 50, "error_rate": 0.0 }, { "coverage": 0.5, "count": 100, "error_rate": 0.0 }, { "coverage": 0.75, "count": 150, "error_rate": 0.0 }, { "coverage": 1.0, "count": 200, "error_rate": 0.0 } ] } } } } } }