{ "benchmark": "LocalLLaMA/typed-decisions", "split": "test", "protocol": "one request per case: state + all five questions (fitted on train)", "mode": "systemone-fitted", "model": "TypeScript/FastBrain-9B", "name": "typescript_fastbrain-9b-fitted", "meta": { "serve": "joint_schema_model.systemone + train-fitted calibration head (L2 reg=0.005 on 1,200 train cases)", "serve_flags": "bfloat16, max_length=16384, fitted_on=train", "errors": 0, "not_supported": false }, "wall_time_s": 37.9412, "summary": { "n_decisions": 2000, "accuracy": 0.807, "kl": 0.08163383829708964, "brier": 0.04374061575809034, "ece": 0.17711654553848563, "accuracy_noul": 0.88, "accuracy_choice": 0.7716666666666666, "accuracy_score": 0.77875, "score_within_1": 0.99875, "score_mae": 0.20525142309461286, "per_workflow": { "agent_trace_observability": { "n_decisions": 500, "accuracy": 0.766, "kl": 0.0770608099372236, "brier": 0.04457242251923376, "ece": 0.20183436014751707, "accuracy_noul": 0.9, "accuracy_choice": 0.725, "accuracy_score": 0.74, "score_within_1": 1.0, "score_mae": 0.1873397418633585 }, "customer_service": { "n_decisions": 500, "accuracy": 0.834, "kl": 0.09237999743266088, "brier": 0.04720350141493786, "ece": 0.17769429981674317, "accuracy_noul": 0.83, "accuracy_choice": 0.815, "accuracy_score": 0.855, "score_within_1": 0.995, "score_mae": 0.16264664335007178 }, "invoice_processing": { "n_decisions": 500, "accuracy": 0.836, "kl": 0.09193783215695348, "brier": 0.04784172228784844, "ece": 0.11770005585374474, "accuracy_noul": 0.93, "accuracy_choice": 0.78, "accuracy_score": 0.77, "score_within_1": 1.0, "score_mae": 0.24064177777075332 }, "security_incidents": { "n_decisions": 500, "accuracy": 0.792, "kl": 0.06515671366152061, "brier": 0.0353448168103413, "ece": 0.21123746633593743, "accuracy_noul": 0.845, "accuracy_choice": 0.77, "accuracy_score": 0.75, "score_within_1": 1.0, "score_mae": 0.23037752939426784 } }, "per_question": { "agent_trace_observability.action": { "n_decisions": 100, "accuracy": 0.73, "kl": 0.09298879511059745, "brier": 0.047588158784912654, "ece": 0.22375233051681045, "accuracy_choice": 0.73 }, "agent_trace_observability.needs_review": { "n_decisions": 100, "accuracy": 0.9, "kl": 0.03739717753519527, "brier": 0.02835461527008, "ece": 0.20054381, "accuracy_noul": 0.9 }, "agent_trace_observability.outcome": { "n_decisions": 100, "accuracy": 0.72, "kl": 0.10743458167624473, "brier": 0.0641251215153878, "ece": 0.17511011265659063, "accuracy_choice": 0.72 }, "agent_trace_observability.risk": { "n_decisions": 100, "accuracy": 0.8, "kl": 0.06663682460279459, "brier": 0.038299156854422735, "ece": 0.23833611536522742, "accuracy_score": 0.8, "score_within_1": 1.0, "score_mae": 0.16805968340872657 }, "agent_trace_observability.urgency": { "n_decisions": 100, "accuracy": 0.68, "kl": 0.08084667076128593, "brier": 0.0444950601713656, "ece": 0.1903288729072122, "accuracy_score": 0.68, "score_within_1": 1.0, "score_mae": 0.2066198003179904 }, "customer_service.action": { "n_decisions": 100, "accuracy": 0.71, "kl": 0.17422497905048193, "brier": 0.08266495383317281, "ece": 0.1494838032265741, "accuracy_choice": 0.71 }, "customer_service.category": { "n_decisions": 100, "accuracy": 0.92, "kl": 0.09275700199700532, "brier": 0.04135654518810614, "ece": 0.09228946626447622, "accuracy_choice": 0.92 }, "customer_service.churn_risk": { "n_decisions": 100, "accuracy": 0.87, "kl": 0.08047298185979117, "brier": 0.038968958222354584, "ece": 0.2952411707552816, "accuracy_score": 0.87, "score_within_1": 1.0, "score_mae": 0.18544014373519857 }, "customer_service.needs_human": { "n_decisions": 100, "accuracy": 0.83, "kl": 0.04646740966659899, "brier": 0.03925850721626, "ece": 0.13978762000000003, "accuracy_noul": 0.83 }, "customer_service.urgency": { "n_decisions": 100, "accuracy": 0.84, "kl": 0.06797761458942707, "brier": 0.03376854261479575, "ece": 0.214303338837384, "accuracy_score": 0.84, "score_within_1": 0.99, "score_mae": 0.13985314296494497 }, "invoice_processing.discrepancy_severity": { "n_decisions": 100, "accuracy": 0.82, "kl": 0.14999327030925053, "brier": 0.0692188752728708, "ece": 0.15126773432369334, "accuracy_score": 0.82, "score_within_1": 1.0, "score_mae": 0.2972442382198659 }, "invoice_processing.disposition": { "n_decisions": 100, "accuracy": 0.78, "kl": 0.13011402368003328, "brier": 0.06934787845525431, "ece": 0.16060270172435623, "accuracy_choice": 0.78 }, "invoice_processing.duplicate": { "n_decisions": 100, "accuracy": 0.9, "kl": 0.03683735423749681, "brier": 0.026522069438619997, "ece": 0.08142392999999995, "accuracy_noul": 0.9 }, "invoice_processing.matches_order": { "n_decisions": 100, "accuracy": 0.96, "kl": 0.054312013671057154, "brier": 0.02630244684444, "ece": 0.10217012000000006, "accuracy_noul": 0.96 }, "invoice_processing.urgency": { "n_decisions": 100, "accuracy": 0.72, "kl": 0.08843249888692961, "brier": 0.0478173414280571, "ece": 0.2008103957431227, "accuracy_score": 0.72, "score_within_1": 1.0, "score_mae": 0.18403931732164075 }, "security_incidents.credential_compromise": { "n_decisions": 100, "accuracy": 0.81, "kl": 0.027662892130680114, "brier": 0.0235511662168, "ece": 0.16027717000000002, "accuracy_noul": 0.81 }, "security_incidents.disposition": { "n_decisions": 100, "accuracy": 0.77, "kl": 0.0930241618392941, "brier": 0.04711140906264851, "ece": 0.24275009450010354, "accuracy_choice": 0.77 }, "security_incidents.severity": { "n_decisions": 100, "accuracy": 0.79, "kl": 0.08313237468836504, "brier": 0.03440369317261647, "ece": 0.30461936357606123, "accuracy_score": 0.79, "score_within_1": 1.0, "score_mae": 0.2187047598865828 }, "security_incidents.true_positive": { "n_decisions": 100, "accuracy": 0.88, "kl": 0.028875774861586857, "brier": 0.02364131343854, "ece": 0.18870840000000003, "accuracy_noul": 0.88 }, "security_incidents.urgency": { "n_decisions": 100, "accuracy": 0.71, "kl": 0.09308836478767697, "brier": 0.04801650216110151, "ece": 0.17481530308799767, "accuracy_score": 0.71, "score_within_1": 1.0, "score_mae": 0.24205029890195293 } } }, "predictions": { "agent_trace_observability_000000": { "action": { "probabilities": { "continue": 0.542038, "human_review": 0.102021, "observe": 0.333859, "stop": 0.022082 } }, "needs_review": { "probabilities": { "false": 0.854698, "true": 0.145302 } }, "outcome": { "probabilities": { "failure": 0.04681, "harmful": 0.0177, "partial": 0.195136, "success": 0.740354 } }, "risk": { "probabilities": { "0": 0.266407, "1": 0.564276, "2": 0.127748, "3": 0.041569 } }, "urgency": { "probabilities": { "0": 0.517619, "1": 0.351549, "2": 0.108014, "3": 0.022818 } } }, "agent_trace_observability_000001": { "action": { "probabilities": { "continue": 0.203965, "human_review": 0.283157, "observe": 0.468914, "stop": 0.043964 } }, "needs_review": { "probabilities": { "false": 0.623712, "true": 0.376288 } }, "outcome": { "probabilities": { "failure": 0.241655, "harmful": 0.016677, "partial": 0.398393, "success": 0.343275 } }, "risk": { "probabilities": { "0": 0.421869, "1": 0.46234, "2": 0.098021, "3": 0.017771 } }, "urgency": { "probabilities": { "0": 0.369415, "1": 0.453194, "2": 0.149362, "3": 0.028029 } } }, "agent_trace_observability_000002": { "action": { "probabilities": { "continue": 0.104842, "human_review": 0.456646, "observe": 0.382194, "stop": 0.056318 } }, "needs_review": { "probabilities": { "false": 0.424135, "true": 0.575865 } }, "outcome": { "probabilities": { "failure": 0.48865, "harmful": 0.016405, "partial": 0.42217, "success": 0.072776 } }, "risk": { "probabilities": { "0": 0.390921, "1": 0.488184, "2": 0.106973, "3": 0.013923 } }, "urgency": { "probabilities": { "0": 0.220884, "1": 0.519258, "2": 0.222507, "3": 0.037351 } } }, "agent_trace_observability_000003": { "action": { "probabilities": { "continue": 0.277412, "human_review": 0.234491, "observe": 0.464356, "stop": 0.023741 } }, "needs_review": { "probabilities": { "false": 0.690235, "true": 0.309765 } }, "outcome": { "probabilities": { "failure": 0.133377, "harmful": 0.010617, "partial": 0.422874, "success": 0.433132 } }, "risk": { "probabilities": { "0": 0.401968, "1": 0.476017, "2": 0.108109, "3": 0.013906 } }, "urgency": { "probabilities": { "0": 0.418237, "1": 0.450601, "2": 0.118, "3": 0.013163 } } }, "agent_trace_observability_000004": { "action": { "probabilities": { "continue": 0.304186, "human_review": 0.216603, "observe": 0.442228, "stop": 0.036983 } }, "needs_review": { "probabilities": { "false": 0.700641, "true": 0.299359 } }, "outcome": { "probabilities": { "failure": 0.234849, "harmful": 0.022353, "partial": 0.474298, "success": 0.268499 } }, "risk": { "probabilities": { "0": 0.42347, "1": 0.478549, "2": 0.084288, "3": 0.013693 } }, "urgency": { "probabilities": { "0": 0.375927, "1": 0.462529, "2": 0.137564, "3": 0.023981 } } }, "agent_trace_observability_000005": { "action": { "probabilities": { "continue": 0.313393, "human_review": 0.202408, "observe": 0.454533, "stop": 0.029666 } }, "needs_review": { "probabilities": { "false": 0.708642, "true": 0.291358 } }, "outcome": { "probabilities": { "failure": 0.105139, "harmful": 0.019209, "partial": 0.379624, "success": 0.496028 } }, "risk": { "probabilities": { "0": 0.204756, "1": 0.628315, "2": 0.136421, "3": 0.030508 } }, "urgency": { "probabilities": { "0": 0.28984, "1": 0.488293, "2": 0.189996, "3": 0.031871 } } }, "agent_trace_observability_000006": { "action": { "probabilities": { "continue": 0.021633, "human_review": 0.608076, "observe": 0.10658, "stop": 0.263711 } }, "needs_review": { "probabilities": { "false": 0.109776, "true": 0.890224 } }, "outcome": { "probabilities": { "failure": 0.11826, "harmful": 0.733581, "partial": 0.11382, "success": 0.034339 } }, "risk": { "probabilities": { "0": 0.015421, "1": 0.047501, "2": 0.227319, "3": 0.709759 } }, "urgency": { "probabilities": { "0": 0.030417, "1": 0.094875, "2": 0.311786, "3": 0.562922 } } }, "agent_trace_observability_000007": { "action": { "probabilities": { "continue": 0.155802, "human_review": 0.364823, "observe": 0.446915, "stop": 0.03246 } }, "needs_review": { "probabilities": { "false": 0.526715, "true": 0.473285 } }, "outcome": { "probabilities": { "failure": 0.251121, "harmful": 0.010146, "partial": 0.503236, "success": 0.235497 } }, "risk": { "probabilities": { "0": 0.296615, "1": 0.588063, "2": 0.103816, "3": 0.011506 } }, "urgency": { "probabilities": { "0": 0.261059, "1": 0.522031, "2": 0.19748, "3": 0.01943 } } }, "agent_trace_observability_000008": { "action": { "probabilities": { "continue": 0.020934, "human_review": 0.633811, "observe": 0.090965, "stop": 0.25429 } }, "needs_review": { "probabilities": { "false": 0.08323, "true": 0.91677 } }, "outcome": { "probabilities": { "failure": 0.148161, "harmful": 0.653803, "partial": 0.171903, "success": 0.026133 } }, "risk": { "probabilities": { "0": 0.013081, "1": 0.061339, "2": 0.293781, "3": 0.631799 } }, "urgency": { "probabilities": { "0": 0.018018, "1": 0.121615, "2": 0.374099, "3": 0.486268 } } }, "agent_trace_observability_000009": { "action": { "probabilities": { "continue": 0.024543, "human_review": 0.610156, "observe": 0.101152, "stop": 0.264149 } }, "needs_review": { "probabilities": { "false": 0.097727, "true": 0.902273 } }, "outcome": { "probabilities": { "failure": 0.099743, "harmful": 0.716865, "partial": 0.157833, "success": 0.025559 } }, "risk": { "probabilities": { "0": 0.008848, "1": 0.054639, "2": 0.267613, "3": 0.6689 } }, "urgency": { "probabilities": { "0": 0.021881, "1": 0.096269, "2": 0.344326, "3": 0.537524 } } }, "agent_trace_observability_000010": { "action": { "probabilities": { "continue": 0.307168, "human_review": 0.198973, "observe": 0.465053, "stop": 0.028806 } }, "needs_review": { "probabilities": { "false": 0.715412, "true": 0.284588 } }, "outcome": { "probabilities": { "failure": 0.153645, "harmful": 0.014193, "partial": 0.494129, "success": 0.338032 } }, "risk": { "probabilities": { "0": 0.23943, "1": 0.625851, "2": 0.119083, "3": 0.015636 } }, "urgency": { "probabilities": { "0": 0.304539, "1": 0.509718, "2": 0.162189, "3": 0.023554 } } }, "agent_trace_observability_000011": { "action": { "probabilities": { "continue": 0.566016, "human_review": 0.089298, "observe": 0.33197, "stop": 0.012716 } }, "needs_review": { "probabilities": { "false": 0.865608, "true": 0.134392 } }, "outcome": { "probabilities": { "failure": 0.050549, "harmful": 0.008433, "partial": 0.261601, "success": 0.679417 } }, "risk": { "probabilities": { "0": 0.596356, "1": 0.348972, "2": 0.047301, "3": 0.007371 } }, "urgency": { "probabilities": { "0": 0.653147, "1": 0.28945, "2": 0.049792, "3": 0.007611 } } }, "agent_trace_observability_000012": { "action": { "probabilities": { "continue": 0.224149, "human_review": 0.332967, "observe": 0.40711, "stop": 0.035774 } }, "needs_review": { "probabilities": { "false": 0.61296, "true": 0.38704 } }, "outcome": { "probabilities": { "failure": 0.095741, "harmful": 0.044276, "partial": 0.394104, "success": 0.465878 } }, "risk": { "probabilities": { "0": 0.20437, "1": 0.300804, "2": 0.418213, "3": 0.076612 } }, "urgency": { "probabilities": { "0": 0.361138, "1": 0.430868, "2": 0.165759, "3": 0.042235 } } }, "agent_trace_observability_000013": { "action": { "probabilities": { "continue": 0.321262, "human_review": 0.197409, "observe": 0.453909, "stop": 0.02742 } }, "needs_review": { "probabilities": { "false": 0.705173, "true": 0.294827 } }, "outcome": { "probabilities": { "failure": 0.156736, "harmful": 0.014229, "partial": 0.522593, "success": 0.306442 } }, "risk": { "probabilities": { "0": 0.344107, "1": 0.559132, "2": 0.086197, "3": 0.010564 } }, "urgency": { "probabilities": { "0": 0.326529, "1": 0.520527, "2": 0.136177, "3": 0.016767 } } }, "agent_trace_observability_000014": { "action": { "probabilities": { "continue": 0.182694, "human_review": 0.33925, "observe": 0.438409, "stop": 0.039647 } }, "needs_review": { "probabilities": { "false": 0.55672, "true": 0.44328 } }, "outcome": { "probabilities": { "failure": 0.202815, "harmful": 0.034665, "partial": 0.448692, "success": 0.313828 } }, "risk": { "probabilities": { "0": 0.199709, "1": 0.549958, "2": 0.192308, "3": 0.058025 } }, "urgency": { "probabilities": { "0": 0.207969, "1": 0.501181, "2": 0.242828, "3": 0.048022 } } }, "agent_trace_observability_000015": { "action": { "probabilities": { "continue": 0.02675, "human_review": 0.559466, "observe": 0.093589, "stop": 0.320195 } }, "needs_review": { "probabilities": { "false": 0.104958, "true": 0.895042 } }, "outcome": { "probabilities": { "failure": 0.129377, "harmful": 0.727727, "partial": 0.120934, "success": 0.021962 } }, "risk": { "probabilities": { "0": 0.012796, "1": 0.046907, "2": 0.267723, "3": 0.672574 } }, "urgency": { "probabilities": { "0": 0.023992, "1": 0.101945, "2": 0.320894, "3": 0.553169 } } }, "agent_trace_observability_000016": { "action": { "probabilities": { "continue": 0.014412, "human_review": 0.706204, "observe": 0.08505, "stop": 0.194334 } }, "needs_review": { "probabilities": { "false": 0.069731, "true": 0.930269 } }, "outcome": { "probabilities": { "failure": 0.230307, "harmful": 0.562162, "partial": 0.185843, "success": 0.021688 } }, "risk": { "probabilities": { "0": 0.015502, "1": 0.083087, "2": 0.350787, "3": 0.550624 } }, "urgency": { "probabilities": { "0": 0.02073, "1": 0.137464, "2": 0.420049, "3": 0.421757 } } }, "agent_trace_observability_000017": { "action": { "probabilities": { "continue": 0.204926, "human_review": 0.365375, "observe": 0.390904, "stop": 0.038794 } }, "needs_review": { "probabilities": { "false": 0.528269, "true": 0.471731 } }, "outcome": { "probabilities": { "failure": 0.129079, "harmful": 0.023798, "partial": 0.382661, "success": 0.464462 } }, "risk": { "probabilities": { "0": 0.081202, "1": 0.387082, "2": 0.466194, "3": 0.065521 } }, "urgency": { "probabilities": { "0": 0.158279, "1": 0.482315, "2": 0.316753, "3": 0.042653 } } }, "agent_trace_observability_000018": { "action": { "probabilities": { "continue": 0.521669, "human_review": 0.144631, "observe": 0.320172, "stop": 0.013528 } }, "needs_review": { "probabilities": { "false": 0.790707, "true": 0.209293 } }, "outcome": { "probabilities": { "failure": 0.102374, "harmful": 0.009062, "partial": 0.400789, "success": 0.487775 } }, "risk": { "probabilities": { "0": 0.728532, "1": 0.221727, "2": 0.042906, "3": 0.006835 } }, "urgency": { "probabilities": { "0": 0.634222, "1": 0.302575, "2": 0.054538, "3": 0.008665 } } }, "agent_trace_observability_000019": { "action": { "probabilities": { "continue": 0.265066, "human_review": 0.290836, "observe": 0.414404, "stop": 0.029693 } }, "needs_review": { "probabilities": { "false": 0.658305, "true": 0.341695 } }, "outcome": { "probabilities": { "failure": 0.112973, "harmful": 0.028758, "partial": 0.404966, "success": 0.453302 } }, "risk": { "probabilities": { "0": 0.259183, "1": 0.332892, "2": 0.362827, "3": 0.045099 } }, "urgency": { "probabilities": { "0": 0.395101, "1": 0.431794, "2": 0.147397, "3": 0.025708 } } }, "agent_trace_observability_000020": { "action": { "probabilities": { "continue": 0.258584, "human_review": 0.303034, "observe": 0.405871, "stop": 0.032511 } }, "needs_review": { "probabilities": { "false": 0.60445, "true": 0.39555 } }, "outcome": { "probabilities": { "failure": 0.05033, "harmful": 0.019991, "partial": 0.270221, "success": 0.659458 } }, "risk": { "probabilities": { "0": 0.080657, "1": 0.293273, "2": 0.551751, "3": 0.074319 } }, "urgency": { "probabilities": { "0": 0.267487, "1": 0.486784, "2": 0.209276, "3": 0.036453 } } }, "agent_trace_observability_000021": { "action": { "probabilities": { "continue": 0.663876, "human_review": 0.076755, "observe": 0.250369, "stop": 0.009 } }, "needs_review": { "probabilities": { "false": 0.884637, "true": 0.115363 } }, "outcome": { "probabilities": { "failure": 0.074483, "harmful": 0.011568, "partial": 0.270961, "success": 0.642988 } }, "risk": { "probabilities": { "0": 0.709032, "1": 0.248855, "2": 0.034522, "3": 0.007591 } }, "urgency": { "probabilities": { "0": 0.690412, "1": 0.267897, "2": 0.037683, "3": 0.004008 } } }, "agent_trace_observability_000022": { "action": { "probabilities": { "continue": 0.328126, "human_review": 0.21666, "observe": 0.422792, "stop": 0.032421 } }, "needs_review": { "probabilities": { "false": 0.700988, "true": 0.299012 } }, "outcome": { "probabilities": { "failure": 0.2626, "harmful": 0.014469, "partial": 0.513423, "success": 0.209508 } }, "risk": { "probabilities": { "0": 0.759103, "1": 0.196783, "2": 0.038145, "3": 0.005969 } }, "urgency": { "probabilities": { "0": 0.526593, "1": 0.363664, "2": 0.09417, "3": 0.015573 } } }, "agent_trace_observability_000023": { "action": { "probabilities": { "continue": 0.385094, "human_review": 0.189261, "observe": 0.38514, "stop": 0.040505 } }, "needs_review": { "probabilities": { "false": 0.762203, "true": 0.237797 } }, "outcome": { "probabilities": { "failure": 0.231007, "harmful": 0.029689, "partial": 0.345733, "success": 0.393571 } }, "risk": { "probabilities": { "0": 0.606219, "1": 0.328749, "2": 0.053496, "3": 0.011536 } }, "urgency": { "probabilities": { "0": 0.576915, "1": 0.321726, "2": 0.077985, "3": 0.023374 } } }, "agent_trace_observability_000024": { "action": { "probabilities": { "continue": 0.070206, "human_review": 0.544923, "observe": 0.320132, "stop": 0.064739 } }, "needs_review": { "probabilities": { "false": 0.339566, "true": 0.660434 } }, "outcome": { "probabilities": { "failure": 0.241486, "harmful": 0.056978, "partial": 0.473851, "success": 0.227686 } }, "risk": { "probabilities": { "0": 0.059611, "1": 0.285147, "2": 0.544017, "3": 0.111225 } }, "urgency": { "probabilities": { "0": 0.113312, "1": 0.412676, "2": 0.380727, "3": 0.093285 } } }, "agent_trace_observability_000025": { "action": { "probabilities": { "continue": 0.018711, "human_review": 0.648258, "observe": 0.096028, "stop": 0.237003 } }, "needs_review": { "probabilities": { "false": 0.08625, "true": 0.91375 } }, "outcome": { "probabilities": { "failure": 0.135345, "harmful": 0.680159, "partial": 0.164928, "success": 0.019568 } }, "risk": { "probabilities": { "0": 0.014088, "1": 0.069704, "2": 0.290593, "3": 0.625615 } }, "urgency": { "probabilities": { "0": 0.022927, "1": 0.111656, "2": 0.377228, "3": 0.488189 } } }, "agent_trace_observability_000026": { "action": { "probabilities": { "continue": 0.012389, "human_review": 0.659926, "observe": 0.080459, "stop": 0.247226 } }, "needs_review": { "probabilities": { "false": 0.063724, "true": 0.936276 } }, "outcome": { "probabilities": { "failure": 0.163358, "harmful": 0.617723, "partial": 0.195178, "success": 0.023741 } }, "risk": { "probabilities": { "0": 0.008055, "1": 0.059671, "2": 0.283278, "3": 0.648996 } }, "urgency": { "probabilities": { "0": 0.014259, "1": 0.092269, "2": 0.35192, "3": 0.541552 } } }, "agent_trace_observability_000027": { "action": { "probabilities": { "continue": 0.270563, "human_review": 0.247628, "observe": 0.445401, "stop": 0.036408 } }, "needs_review": { "probabilities": { "false": 0.652555, "true": 0.347445 } }, "outcome": { "probabilities": { "failure": 0.162282, "harmful": 0.023206, "partial": 0.457145, "success": 0.357367 } }, "risk": { "probabilities": { "0": 0.248692, "1": 0.582191, "2": 0.140494, "3": 0.028623 } }, "urgency": { "probabilities": { "0": 0.23883, "1": 0.509414, "2": 0.215692, "3": 0.036064 } } }, "agent_trace_observability_000028": { "action": { "probabilities": { "continue": 0.584495, "human_review": 0.085606, "observe": 0.321252, "stop": 0.008647 } }, "needs_review": { "probabilities": { "false": 0.873241, "true": 0.126759 } }, "outcome": { "probabilities": { "failure": 0.043186, "harmful": 0.005542, "partial": 0.240862, "success": 0.71041 } }, "risk": { "probabilities": { "0": 0.78171, "1": 0.186489, "2": 0.027419, "3": 0.004382 } }, "urgency": { "probabilities": { "0": 0.740616, "1": 0.220731, "2": 0.03421, "3": 0.004443 } } }, "agent_trace_observability_000029": { "action": { "probabilities": { "continue": 0.148005, "human_review": 0.40557, "observe": 0.404918, "stop": 0.041507 } }, "needs_review": { "probabilities": { "false": 0.494919, "true": 0.505081 } }, "outcome": { "probabilities": { "failure": 0.403143, "harmful": 0.014177, "partial": 0.455018, "success": 0.127662 } }, "risk": { "probabilities": { "0": 0.279835, "1": 0.571134, "2": 0.133776, "3": 0.015255 } }, "urgency": { "probabilities": { "0": 0.256816, "1": 0.496038, "2": 0.219037, "3": 0.028109 } } }, "agent_trace_observability_000030": { "action": { "probabilities": { "continue": 0.075525, "human_review": 0.538796, "observe": 0.334991, "stop": 0.050688 } }, "needs_review": { "probabilities": { "false": 0.343313, "true": 0.656687 } }, "outcome": { "probabilities": { "failure": 0.219761, "harmful": 0.030923, "partial": 0.518435, "success": 0.230881 } }, "risk": { "probabilities": { "0": 0.079338, "1": 0.361813, "2": 0.493203, "3": 0.065647 } }, "urgency": { "probabilities": { "0": 0.111473, "1": 0.502867, "2": 0.332897, "3": 0.052763 } } }, "agent_trace_observability_000031": { "action": { "probabilities": { "continue": 0.098105, "human_review": 0.485783, "observe": 0.342737, "stop": 0.073375 } }, "needs_review": { "probabilities": { "false": 0.390001, "true": 0.609999 } }, "outcome": { "probabilities": { "failure": 0.586802, "harmful": 0.018168, "partial": 0.343458, "success": 0.051572 } }, "risk": { "probabilities": { "0": 0.353817, "1": 0.478456, "2": 0.147046, "3": 0.020681 } }, "urgency": { "probabilities": { "0": 0.198322, "1": 0.48818, "2": 0.259026, "3": 0.054472 } } }, "agent_trace_observability_000032": { "action": { "probabilities": { "continue": 0.031654, "human_review": 0.698226, "observe": 0.133856, "stop": 0.136264 } }, "needs_review": { "probabilities": { "false": 0.127811, "true": 0.872189 } }, "outcome": { "probabilities": { "failure": 0.188515, "harmful": 0.496638, "partial": 0.278356, "success": 0.036492 } }, "risk": { "probabilities": { "0": 0.05709, "1": 0.121169, "2": 0.354409, "3": 0.467332 } }, "urgency": { "probabilities": { "0": 0.075259, "1": 0.240129, "2": 0.393555, "3": 0.291057 } } }, "agent_trace_observability_000033": { "action": { "probabilities": { "continue": 0.029558, "human_review": 0.667316, "observe": 0.10958, "stop": 0.193546 } }, "needs_review": { "probabilities": { "false": 0.100656, "true": 0.899344 } }, "outcome": { "probabilities": { "failure": 0.106802, "harmful": 0.657728, "partial": 0.194734, "success": 0.040736 } }, "risk": { "probabilities": { "0": 0.017177, "1": 0.075305, "2": 0.335709, "3": 0.571809 } }, "urgency": { "probabilities": { "0": 0.029269, "1": 0.146987, "2": 0.386144, "3": 0.437601 } } }, "agent_trace_observability_000034": { "action": { "probabilities": { "continue": 0.133123, "human_review": 0.421199, "observe": 0.403548, "stop": 0.04213 } }, "needs_review": { "probabilities": { "false": 0.493057, "true": 0.506943 } }, "outcome": { "probabilities": { "failure": 0.114345, "harmful": 0.035886, "partial": 0.344393, "success": 0.505376 } }, "risk": { "probabilities": { "0": 0.063955, "1": 0.324433, "2": 0.514782, "3": 0.09683 } }, "urgency": { "probabilities": { "0": 0.178742, "1": 0.435677, "2": 0.323871, "3": 0.061711 } } }, "agent_trace_observability_000035": { "action": { "probabilities": { "continue": 0.267491, "human_review": 0.232691, "observe": 0.476027, "stop": 0.023791 } }, "needs_review": { "probabilities": { "false": 0.674667, "true": 0.325333 } }, "outcome": { "probabilities": { "failure": 0.166792, "harmful": 0.009612, "partial": 0.515572, "success": 0.308024 } }, "risk": { "probabilities": { "0": 0.503608, "1": 0.412413, "2": 0.075841, "3": 0.008138 } }, "urgency": { "probabilities": { "0": 0.401253, "1": 0.458929, "2": 0.124694, "3": 0.015123 } } }, "agent_trace_observability_000036": { "action": { "probabilities": { "continue": 0.176083, "human_review": 0.362101, "observe": 0.41495, "stop": 0.046866 } }, "needs_review": { "probabilities": { "false": 0.500873, "true": 0.499127 } }, "outcome": { "probabilities": { "failure": 0.334189, "harmful": 0.012709, "partial": 0.510369, "success": 0.142733 } }, "risk": { "probabilities": { "0": 0.55118, "1": 0.363539, "2": 0.075667, "3": 0.009614 } }, "urgency": { "probabilities": { "0": 0.248451, "1": 0.520496, "2": 0.198527, "3": 0.032526 } } }, "agent_trace_observability_000037": { "action": { "probabilities": { "continue": 0.018828, "human_review": 0.647838, "observe": 0.113341, "stop": 0.219993 } }, "needs_review": { "probabilities": { "false": 0.098505, "true": 0.901495 } }, "outcome": { "probabilities": { "failure": 0.118413, "harmful": 0.681653, "partial": 0.169366, "success": 0.030568 } }, "risk": { "probabilities": { "0": 0.013358, "1": 0.067086, "2": 0.288765, "3": 0.630791 } }, "urgency": { "probabilities": { "0": 0.02731, "1": 0.107008, "2": 0.361636, "3": 0.504046 } } }, "agent_trace_observability_000038": { "action": { "probabilities": { "continue": 0.17045, "human_review": 0.293968, "observe": 0.513856, "stop": 0.021726 } }, "needs_review": { "probabilities": { "false": 0.611302, "true": 0.388698 } }, "outcome": { "probabilities": { "failure": 0.172146, "harmful": 0.008218, "partial": 0.420964, "success": 0.398672 } }, "risk": { "probabilities": { "0": 0.427856, "1": 0.463663, "2": 0.098989, "3": 0.009492 } }, "urgency": { "probabilities": { "0": 0.410245, "1": 0.442816, "2": 0.133533, "3": 0.013407 } } }, "agent_trace_observability_000039": { "action": { "probabilities": { "continue": 0.02854, "human_review": 0.615346, "observe": 0.114059, "stop": 0.242055 } }, "needs_review": { "probabilities": { "false": 0.120168, "true": 0.879832 } }, "outcome": { "probabilities": { "failure": 0.072414, "harmful": 0.756884, "partial": 0.125649, "success": 0.045053 } }, "risk": { "probabilities": { "0": 0.008758, "1": 0.045364, "2": 0.262616, "3": 0.683262 } }, "urgency": { "probabilities": { "0": 0.027319, "1": 0.087324, "2": 0.282635, "3": 0.602722 } } }, "agent_trace_observability_000040": { "action": { "probabilities": { "continue": 0.109489, "human_review": 0.452757, "observe": 0.387916, "stop": 0.049838 } }, "needs_review": { "probabilities": { "false": 0.43858, "true": 0.56142 } }, "outcome": { "probabilities": { "failure": 0.460988, "harmful": 0.013768, "partial": 0.425514, "success": 0.09973 } }, "risk": { "probabilities": { "0": 0.276157, "1": 0.576047, "2": 0.131547, "3": 0.016249 } }, "urgency": { "probabilities": { "0": 0.193951, "1": 0.528483, "2": 0.241632, "3": 0.035934 } } }, "agent_trace_observability_000041": { "action": { "probabilities": { "continue": 0.257481, "human_review": 0.224678, "observe": 0.498135, "stop": 0.019706 } }, "needs_review": { "probabilities": { "false": 0.705403, "true": 0.294597 } }, "outcome": { "probabilities": { "failure": 0.165646, "harmful": 0.009917, "partial": 0.495654, "success": 0.328784 } }, "risk": { "probabilities": { "0": 0.628581, "1": 0.307164, "2": 0.056827, "3": 0.007428 } }, "urgency": { "probabilities": { "0": 0.493455, "1": 0.400491, "2": 0.095971, "3": 0.010083 } } }, "agent_trace_observability_000042": { "action": { "probabilities": { "continue": 0.177738, "human_review": 0.352707, "observe": 0.423737, "stop": 0.045818 } }, "needs_review": { "probabilities": { "false": 0.51235, "true": 0.48765 } }, "outcome": { "probabilities": { "failure": 0.294507, "harmful": 0.013175, "partial": 0.529669, "success": 0.162649 } }, "risk": { "probabilities": { "0": 0.454597, "1": 0.441363, "2": 0.091871, "3": 0.012169 } }, "urgency": { "probabilities": { "0": 0.238096, "1": 0.52228, "2": 0.206942, "3": 0.032682 } } }, "agent_trace_observability_000043": { "action": { "probabilities": { "continue": 0.066037, "human_review": 0.554257, "observe": 0.265926, "stop": 0.11378 } }, "needs_review": { "probabilities": { "false": 0.278894, "true": 0.721106 } }, "outcome": { "probabilities": { "failure": 0.247246, "harmful": 0.126518, "partial": 0.433047, "success": 0.193189 } }, "risk": { "probabilities": { "0": 0.033696, "1": 0.162925, "2": 0.603604, "3": 0.199775 } }, "urgency": { "probabilities": { "0": 0.079639, "1": 0.292647, "2": 0.419874, "3": 0.20784 } } }, "agent_trace_observability_000044": { "action": { "probabilities": { "continue": 0.088067, "human_review": 0.527572, "observe": 0.305537, "stop": 0.078824 } }, "needs_review": { "probabilities": { "false": 0.363956, "true": 0.636044 } }, "outcome": { "probabilities": { "failure": 0.329572, "harmful": 0.060834, "partial": 0.476258, "success": 0.133336 } }, "risk": { "probabilities": { "0": 0.129489, "1": 0.29958, "2": 0.452787, "3": 0.118145 } }, "urgency": { "probabilities": { "0": 0.145363, "1": 0.419643, "2": 0.336715, "3": 0.098279 } } }, "agent_trace_observability_000045": { "action": { "probabilities": { "continue": 0.222267, "human_review": 0.243839, "observe": 0.508178, "stop": 0.025716 } }, "needs_review": { "probabilities": { "false": 0.661581, "true": 0.338419 } }, "outcome": { "probabilities": { "failure": 0.177725, "harmful": 0.012357, "partial": 0.460802, "success": 0.349116 } }, "risk": { "probabilities": { "0": 0.391215, "1": 0.509248, "2": 0.089149, "3": 0.010388 } }, "urgency": { "probabilities": { "0": 0.359779, "1": 0.486239, "2": 0.13782, "3": 0.016162 } } }, "agent_trace_observability_000046": { "action": { "probabilities": { "continue": 0.54793, "human_review": 0.093141, "observe": 0.347826, "stop": 0.011103 } }, "needs_review": { "probabilities": { "false": 0.875618, "true": 0.124382 } }, "outcome": { "probabilities": { "failure": 0.050719, "harmful": 0.009258, "partial": 0.240505, "success": 0.699518 } }, "risk": { "probabilities": { "0": 0.74648, "1": 0.20826, "2": 0.037412, "3": 0.007848 } }, "urgency": { "probabilities": { "0": 0.725456, "1": 0.230213, "2": 0.038711, "3": 0.00562 } } }, "agent_trace_observability_000047": { "action": { "probabilities": { "continue": 0.161931, "human_review": 0.297419, "observe": 0.509289, "stop": 0.031361 } }, "needs_review": { "probabilities": { "false": 0.608736, "true": 0.391264 } }, "outcome": { "probabilities": { "failure": 0.188021, "harmful": 0.015777, "partial": 0.413827, "success": 0.382376 } }, "risk": { "probabilities": { "0": 0.258634, "1": 0.60255, "2": 0.117468, "3": 0.021348 } }, "urgency": { "probabilities": { "0": 0.315364, "1": 0.463787, "2": 0.192735, "3": 0.028114 } } }, "agent_trace_observability_000048": { "action": { "probabilities": { "continue": 0.255994, "human_review": 0.251142, "observe": 0.4615, "stop": 0.031365 } }, "needs_review": { "probabilities": { "false": 0.648184, "true": 0.351816 } }, "outcome": { "probabilities": { "failure": 0.166824, "harmful": 0.014138, "partial": 0.505604, "success": 0.313434 } }, "risk": { "probabilities": { "0": 0.227863, "1": 0.619124, "2": 0.1369, "3": 0.016113 } }, "urgency": { "probabilities": { "0": 0.251581, "1": 0.537722, "2": 0.185917, "3": 0.02478 } } }, "agent_trace_observability_000049": { "action": { "probabilities": { "continue": 0.186278, "human_review": 0.37116, "observe": 0.379446, "stop": 0.063117 } }, "needs_review": { "probabilities": { "false": 0.552026, "true": 0.447974 } }, "outcome": { "probabilities": { "failure": 0.086752, "harmful": 0.063731, "partial": 0.283795, "success": 0.565722 } }, "risk": { "probabilities": { "0": 0.089957, "1": 0.222864, "2": 0.552826, "3": 0.134353 } }, "urgency": { "probabilities": { "0": 0.254023, "1": 0.414686, "2": 0.246735, "3": 0.084555 } } }, "agent_trace_observability_000050": { "action": { "probabilities": { "continue": 0.122481, "human_review": 0.459104, "observe": 0.381588, "stop": 0.036827 } }, "needs_review": { "probabilities": { "false": 0.449905, "true": 0.550095 } }, "outcome": { "probabilities": { "failure": 0.14766, "harmful": 0.024895, "partial": 0.463121, "success": 0.364324 } }, "risk": { "probabilities": { "0": 0.076941, "1": 0.365895, "2": 0.494768, "3": 0.062396 } }, "urgency": { "probabilities": { "0": 0.161971, "1": 0.513941, "2": 0.289285, "3": 0.034803 } } }, "agent_trace_observability_000051": { "action": { "probabilities": { "continue": 0.224987, "human_review": 0.313756, "observe": 0.433462, "stop": 0.027795 } }, "needs_review": { "probabilities": { "false": 0.590976, "true": 0.409024 } }, "outcome": { "probabilities": { "failure": 0.326949, "harmful": 0.009343, "partial": 0.534735, "success": 0.128973 } }, "risk": { "probabilities": { "0": 0.729293, "1": 0.225645, "2": 0.0408, "3": 0.004262 } }, "urgency": { "probabilities": { "0": 0.43119, "1": 0.43887, "2": 0.115963, "3": 0.013976 } } }, "agent_trace_observability_000052": { "action": { "probabilities": { "continue": 0.183197, "human_review": 0.394453, "observe": 0.382012, "stop": 0.040338 } }, "needs_review": { "probabilities": { "false": 0.544824, "true": 0.455176 } }, "outcome": { "probabilities": { "failure": 0.143123, "harmful": 0.042221, "partial": 0.384621, "success": 0.430035 } }, "risk": { "probabilities": { "0": 0.240572, "1": 0.257515, "2": 0.435752, "3": 0.066162 } }, "urgency": { "probabilities": { "0": 0.349206, "1": 0.418276, "2": 0.190549, "3": 0.04197 } } }, "agent_trace_observability_000053": { "action": { "probabilities": { "continue": 0.134369, "human_review": 0.382752, "observe": 0.445575, "stop": 0.037304 } }, "needs_review": { "probabilities": { "false": 0.507459, "true": 0.492541 } }, "outcome": { "probabilities": { "failure": 0.304873, "harmful": 0.011246, "partial": 0.484298, "success": 0.199583 } }, "risk": { "probabilities": { "0": 0.311238, "1": 0.571696, "2": 0.104093, "3": 0.012973 } }, "urgency": { "probabilities": { "0": 0.224994, "1": 0.533054, "2": 0.215487, "3": 0.026465 } } }, "agent_trace_observability_000054": { "action": { "probabilities": { "continue": 0.15683, "human_review": 0.334312, "observe": 0.476322, "stop": 0.032535 } }, "needs_review": { "probabilities": { "false": 0.560373, "true": 0.439627 } }, "outcome": { "probabilities": { "failure": 0.205162, "harmful": 0.011035, "partial": 0.451422, "success": 0.332381 } }, "risk": { "probabilities": { "0": 0.283527, "1": 0.574384, "2": 0.125459, "3": 0.01663 } }, "urgency": { "probabilities": { "0": 0.305287, "1": 0.490673, "2": 0.181912, "3": 0.022128 } } }, "agent_trace_observability_000055": { "action": { "probabilities": { "continue": 0.564723, "human_review": 0.085311, "observe": 0.33483, "stop": 0.015136 } }, "needs_review": { "probabilities": { "false": 0.86835, "true": 0.13165 } }, "outcome": { "probabilities": { "failure": 0.052529, "harmful": 0.01003, "partial": 0.257575, "success": 0.679866 } }, "risk": { "probabilities": { "0": 0.417789, "1": 0.498795, "2": 0.072387, "3": 0.011029 } }, "urgency": { "probabilities": { "0": 0.603588, "1": 0.328085, "2": 0.060647, "3": 0.00768 } } }, "agent_trace_observability_000056": { "action": { "probabilities": { "continue": 0.405457, "human_review": 0.196212, "observe": 0.375535, "stop": 0.022796 } }, "needs_review": { "probabilities": { "false": 0.759006, "true": 0.240994 } }, "outcome": { "probabilities": { "failure": 0.133906, "harmful": 0.017786, "partial": 0.316093, "success": 0.532215 } }, "risk": { "probabilities": { "0": 0.2993, "1": 0.560388, "2": 0.118068, "3": 0.022244 } }, "urgency": { "probabilities": { "0": 0.448345, "1": 0.406321, "2": 0.126573, "3": 0.018761 } } }, "agent_trace_observability_000057": { "action": { "probabilities": { "continue": 0.148834, "human_review": 0.308604, "observe": 0.510081, "stop": 0.032481 } }, "needs_review": { "probabilities": { "false": 0.577411, "true": 0.422589 } }, "outcome": { "probabilities": { "failure": 0.192152, "harmful": 0.014616, "partial": 0.482732, "success": 0.3105 } }, "risk": { "probabilities": { "0": 0.261137, "1": 0.551497, "2": 0.162616, "3": 0.02475 } }, "urgency": { "probabilities": { "0": 0.331675, "1": 0.453938, "2": 0.183981, "3": 0.030406 } } }, "agent_trace_observability_000058": { "action": { "probabilities": { "continue": 0.18778, "human_review": 0.31895, "observe": 0.456698, "stop": 0.036572 } }, "needs_review": { "probabilities": { "false": 0.553394, "true": 0.446606 } }, "outcome": { "probabilities": { "failure": 0.2059, "harmful": 0.016256, "partial": 0.506778, "success": 0.271066 } }, "risk": { "probabilities": { "0": 0.238783, "1": 0.562572, "2": 0.174389, "3": 0.024256 } }, "urgency": { "probabilities": { "0": 0.262302, "1": 0.501138, "2": 0.202999, "3": 0.033561 } } }, "agent_trace_observability_000059": { "action": { "probabilities": { "continue": 0.013625, "human_review": 0.675465, "observe": 0.072015, "stop": 0.238895 } }, "needs_review": { "probabilities": { "false": 0.067254, "true": 0.932746 } }, "outcome": { "probabilities": { "failure": 0.121372, "harmful": 0.764257, "partial": 0.094586, "success": 0.019785 } }, "risk": { "probabilities": { "0": 0.007522, "1": 0.047065, "2": 0.234217, "3": 0.711196 } }, "urgency": { "probabilities": { "0": 0.01403, "1": 0.077921, "2": 0.341949, "3": 0.5661 } } }, "agent_trace_observability_000060": { "action": { "probabilities": { "continue": 0.437229, "human_review": 0.129121, "observe": 0.418548, "stop": 0.015102 } }, "needs_review": { "probabilities": { "false": 0.829175, "true": 0.170825 } }, "outcome": { "probabilities": { "failure": 0.06692, "harmful": 0.009392, "partial": 0.261011, "success": 0.662677 } }, "risk": { "probabilities": { "0": 0.426316, "1": 0.502739, "2": 0.061867, "3": 0.009078 } }, "urgency": { "probabilities": { "0": 0.586737, "1": 0.341156, "2": 0.064874, "3": 0.007233 } } }, "agent_trace_observability_000061": { "action": { "probabilities": { "continue": 0.085243, "human_review": 0.487452, "observe": 0.368832, "stop": 0.058472 } }, "needs_review": { "probabilities": { "false": 0.406627, "true": 0.593373 } }, "outcome": { "probabilities": { "failure": 0.169916, "harmful": 0.051381, "partial": 0.441313, "success": 0.33739 } }, "risk": { "probabilities": { "0": 0.037575, "1": 0.284337, "2": 0.559322, "3": 0.118766 } }, "urgency": { "probabilities": { "0": 0.128504, "1": 0.400932, "2": 0.381264, "3": 0.089299 } } }, "agent_trace_observability_000062": { "action": { "probabilities": { "continue": 0.243288, "human_review": 0.25713, "observe": 0.469394, "stop": 0.030188 } }, "needs_review": { "probabilities": { "false": 0.640305, "true": 0.359695 } }, "outcome": { "probabilities": { "failure": 0.190988, "harmful": 0.011042, "partial": 0.526376, "success": 0.271594 } }, "risk": { "probabilities": { "0": 0.48371, "1": 0.4252, "2": 0.081882, "3": 0.009208 } }, "urgency": { "probabilities": { "0": 0.377474, "1": 0.475946, "2": 0.129929, "3": 0.016652 } } }, "agent_trace_observability_000063": { "action": { "probabilities": { "continue": 0.02472, "human_review": 0.656573, "observe": 0.089205, "stop": 0.229502 } }, "needs_review": { "probabilities": { "false": 0.097941, "true": 0.902059 } }, "outcome": { "probabilities": { "failure": 0.12123, "harmful": 0.757096, "partial": 0.101961, "success": 0.019713 } }, "risk": { "probabilities": { "0": 0.012013, "1": 0.050284, "2": 0.261883, "3": 0.67582 } }, "urgency": { "probabilities": { "0": 0.021378, "1": 0.107129, "2": 0.346149, "3": 0.525344 } } }, "agent_trace_observability_000064": { "action": { "probabilities": { "continue": 0.258416, "human_review": 0.314437, "observe": 0.393859, "stop": 0.033287 } }, "needs_review": { "probabilities": { "false": 0.573136, "true": 0.426864 } }, "outcome": { "probabilities": { "failure": 0.257907, "harmful": 0.016713, "partial": 0.566882, "success": 0.158498 } }, "risk": { "probabilities": { "0": 0.527134, "1": 0.357529, "2": 0.099066, "3": 0.016271 } }, "urgency": { "probabilities": { "0": 0.341132, "1": 0.473696, "2": 0.158138, "3": 0.027034 } } }, "agent_trace_observability_000065": { "action": { "probabilities": { "continue": 0.024416, "human_review": 0.657861, "observe": 0.101591, "stop": 0.216132 } }, "needs_review": { "probabilities": { "false": 0.09939, "true": 0.90061 } }, "outcome": { "probabilities": { "failure": 0.109214, "harmful": 0.707312, "partial": 0.152803, "success": 0.030671 } }, "risk": { "probabilities": { "0": 0.009008, "1": 0.066757, "2": 0.275638, "3": 0.648597 } }, "urgency": { "probabilities": { "0": 0.018731, "1": 0.104799, "2": 0.39862, "3": 0.47785 } } }, "agent_trace_observability_000066": { "action": { "probabilities": { "continue": 0.140142, "human_review": 0.360795, "observe": 0.464861, "stop": 0.034203 } }, "needs_review": { "probabilities": { "false": 0.515556, "true": 0.484444 } }, "outcome": { "probabilities": { "failure": 0.287584, "harmful": 0.010178, "partial": 0.485051, "success": 0.217187 } }, "risk": { "probabilities": { "0": 0.477997, "1": 0.434285, "2": 0.078828, "3": 0.008889 } }, "urgency": { "probabilities": { "0": 0.30496, "1": 0.501301, "2": 0.173494, "3": 0.020245 } } }, "agent_trace_observability_000067": { "action": { "probabilities": { "continue": 0.606234, "human_review": 0.070062, "observe": 0.314778, "stop": 0.008926 } }, "needs_review": { "probabilities": { "false": 0.895583, "true": 0.104417 } }, "outcome": { "probabilities": { "failure": 0.042712, "harmful": 0.007562, "partial": 0.265024, "success": 0.684702 } }, "risk": { "probabilities": { "0": 0.74589, "1": 0.223334, "2": 0.025897, "3": 0.004879 } }, "urgency": { "probabilities": { "0": 0.761762, "1": 0.206959, "2": 0.027416, "3": 0.003863 } } }, "agent_trace_observability_000068": { "action": { "probabilities": { "continue": 0.165977, "human_review": 0.394226, "observe": 0.385348, "stop": 0.05445 } }, "needs_review": { "probabilities": { "false": 0.459429, "true": 0.540571 } }, "outcome": { "probabilities": { "failure": 0.364956, "harmful": 0.013094, "partial": 0.489205, "success": 0.132745 } }, "risk": { "probabilities": { "0": 0.457241, "1": 0.407312, "2": 0.121398, "3": 0.014049 } }, "urgency": { "probabilities": { "0": 0.223133, "1": 0.519759, "2": 0.219804, "3": 0.037304 } } }, "agent_trace_observability_000069": { "action": { "probabilities": { "continue": 0.014867, "human_review": 0.670005, "observe": 0.08645, "stop": 0.228678 } }, "needs_review": { "probabilities": { "false": 0.070805, "true": 0.929195 } }, "outcome": { "probabilities": { "failure": 0.144829, "harmful": 0.642368, "partial": 0.191216, "success": 0.021587 } }, "risk": { "probabilities": { "0": 0.008993, "1": 0.056675, "2": 0.277412, "3": 0.65692 } }, "urgency": { "probabilities": { "0": 0.015147, "1": 0.089235, "2": 0.346012, "3": 0.549606 } } }, "agent_trace_observability_000070": { "action": { "probabilities": { "continue": 0.223267, "human_review": 0.359147, "observe": 0.383094, "stop": 0.034493 } }, "needs_review": { "probabilities": { "false": 0.556938, "true": 0.443062 } }, "outcome": { "probabilities": { "failure": 0.07029, "harmful": 0.021174, "partial": 0.312404, "success": 0.596132 } }, "risk": { "probabilities": { "0": 0.120381, "1": 0.294556, "2": 0.529419, "3": 0.055644 } }, "urgency": { "probabilities": { "0": 0.282814, "1": 0.471241, "2": 0.210468, "3": 0.035477 } } }, "agent_trace_observability_000071": { "action": { "probabilities": { "continue": 0.283473, "human_review": 0.259035, "observe": 0.42975, "stop": 0.027742 } }, "needs_review": { "probabilities": { "false": 0.671319, "true": 0.328681 } }, "outcome": { "probabilities": { "failure": 0.204967, "harmful": 0.021454, "partial": 0.393953, "success": 0.379627 } }, "risk": { "probabilities": { "0": 0.301767, "1": 0.560423, "2": 0.116964, "3": 0.020846 } }, "urgency": { "probabilities": { "0": 0.34019, "1": 0.501937, "2": 0.142639, "3": 0.015234 } } }, "agent_trace_observability_000072": { "action": { "probabilities": { "continue": 0.095351, "human_review": 0.526472, "observe": 0.316152, "stop": 0.062025 } }, "needs_review": { "probabilities": { "false": 0.346592, "true": 0.653408 } }, "outcome": { "probabilities": { "failure": 0.214876, "harmful": 0.049179, "partial": 0.514526, "success": 0.221419 } }, "risk": { "probabilities": { "0": 0.079682, "1": 0.258834, "2": 0.538746, "3": 0.122738 } }, "urgency": { "probabilities": { "0": 0.113308, "1": 0.427024, "2": 0.362823, "3": 0.096844 } } }, "agent_trace_observability_000073": { "action": { "probabilities": { "continue": 0.130973, "human_review": 0.446508, "observe": 0.366092, "stop": 0.056427 } }, "needs_review": { "probabilities": { "false": 0.449849, "true": 0.550151 } }, "outcome": { "probabilities": { "failure": 0.243415, "harmful": 0.033838, "partial": 0.438792, "success": 0.283955 } }, "risk": { "probabilities": { "0": 0.159562, "1": 0.331232, "2": 0.444756, "3": 0.06445 } }, "urgency": { "probabilities": { "0": 0.192518, "1": 0.466646, "2": 0.281208, "3": 0.059628 } } }, "agent_trace_observability_000074": { "action": { "probabilities": { "continue": 0.461503, "human_review": 0.149969, "observe": 0.366421, "stop": 0.022107 } }, "needs_review": { "probabilities": { "false": 0.803634, "true": 0.196366 } }, "outcome": { "probabilities": { "failure": 0.12694, "harmful": 0.016181, "partial": 0.328521, "success": 0.528358 } }, "risk": { "probabilities": { "0": 0.62816, "1": 0.313341, "2": 0.049758, "3": 0.008741 } }, "urgency": { "probabilities": { "0": 0.577901, "1": 0.338166, "2": 0.070762, "3": 0.013171 } } }, "agent_trace_observability_000075": { "action": { "probabilities": { "continue": 0.280618, "human_review": 0.185404, "observe": 0.510475, "stop": 0.023503 } }, "needs_review": { "probabilities": { "false": 0.738266, "true": 0.261734 } }, "outcome": { "probabilities": { "failure": 0.121858, "harmful": 0.015348, "partial": 0.429744, "success": 0.43305 } }, "risk": { "probabilities": { "0": 0.281892, "1": 0.620434, "2": 0.084918, "3": 0.012756 } }, "urgency": { "probabilities": { "0": 0.40599, "1": 0.464376, "2": 0.115706, "3": 0.013928 } } }, "agent_trace_observability_000076": { "action": { "probabilities": { "continue": 0.138418, "human_review": 0.451836, "observe": 0.360309, "stop": 0.049437 } }, "needs_review": { "probabilities": { "false": 0.453931, "true": 0.546069 } }, "outcome": { "probabilities": { "failure": 0.208165, "harmful": 0.055826, "partial": 0.530895, "success": 0.205114 } }, "risk": { "probabilities": { "0": 0.188386, "1": 0.279754, "2": 0.455742, "3": 0.076119 } }, "urgency": { "probabilities": { "0": 0.230455, "1": 0.450564, "2": 0.259474, "3": 0.059508 } } }, "agent_trace_observability_000077": { "action": { "probabilities": { "continue": 0.035613, "human_review": 0.669845, "observe": 0.142924, "stop": 0.151618 } }, "needs_review": { "probabilities": { "false": 0.132141, "true": 0.867859 } }, "outcome": { "probabilities": { "failure": 0.151836, "harmful": 0.52474, "partial": 0.290176, "success": 0.033248 } }, "risk": { "probabilities": { "0": 0.025837, "1": 0.097957, "2": 0.311074, "3": 0.565132 } }, "urgency": { "probabilities": { "0": 0.047817, "1": 0.169524, "2": 0.381518, "3": 0.401141 } } }, "agent_trace_observability_000078": { "action": { "probabilities": { "continue": 0.071178, "human_review": 0.516306, "observe": 0.351434, "stop": 0.061082 } }, "needs_review": { "probabilities": { "false": 0.364603, "true": 0.635397 } }, "outcome": { "probabilities": { "failure": 0.1775, "harmful": 0.057512, "partial": 0.449219, "success": 0.315768 } }, "risk": { "probabilities": { "0": 0.036859, "1": 0.269627, "2": 0.569837, "3": 0.123677 } }, "urgency": { "probabilities": { "0": 0.117865, "1": 0.398239, "2": 0.383145, "3": 0.10075 } } }, "agent_trace_observability_000079": { "action": { "probabilities": { "continue": 0.61404, "human_review": 0.069931, "observe": 0.304, "stop": 0.012029 } }, "needs_review": { "probabilities": { "false": 0.891131, "true": 0.108869 } }, "outcome": { "probabilities": { "failure": 0.041578, "harmful": 0.00972, "partial": 0.252889, "success": 0.695813 } }, "risk": { "probabilities": { "0": 0.461002, "1": 0.47375, "2": 0.054995, "3": 0.010254 } }, "urgency": { "probabilities": { "0": 0.653493, "1": 0.294301, "2": 0.04527, "3": 0.006936 } } }, "agent_trace_observability_000080": { "action": { "probabilities": { "continue": 0.128196, "human_review": 0.444931, "observe": 0.327498, "stop": 0.099375 } }, "needs_review": { "probabilities": { "false": 0.437331, "true": 0.562669 } }, "outcome": { "probabilities": { "failure": 0.327915, "harmful": 0.115701, "partial": 0.426042, "success": 0.130342 } }, "risk": { "probabilities": { "0": 0.144606, "1": 0.213038, "2": 0.490485, "3": 0.151871 } }, "urgency": { "probabilities": { "0": 0.193487, "1": 0.39721, "2": 0.289801, "3": 0.119502 } } }, "agent_trace_observability_000081": { "action": { "probabilities": { "continue": 0.166178, "human_review": 0.335211, "observe": 0.455158, "stop": 0.043453 } }, "needs_review": { "probabilities": { "false": 0.533506, "true": 0.466494 } }, "outcome": { "probabilities": { "failure": 0.23107, "harmful": 0.023328, "partial": 0.494287, "success": 0.251315 } }, "risk": { "probabilities": { "0": 0.25119, "1": 0.538584, "2": 0.178256, "3": 0.03197 } }, "urgency": { "probabilities": { "0": 0.251706, "1": 0.484473, "2": 0.220201, "3": 0.04362 } } }, "agent_trace_observability_000082": { "action": { "probabilities": { "continue": 0.161402, "human_review": 0.404608, "observe": 0.370482, "stop": 0.063508 } }, "needs_review": { "probabilities": { "false": 0.475921, "true": 0.524079 } }, "outcome": { "probabilities": { "failure": 0.55561, "harmful": 0.013463, "partial": 0.38939, "success": 0.041537 } }, "risk": { "probabilities": { "0": 0.777374, "1": 0.174289, "2": 0.042436, "3": 0.005901 } }, "urgency": { "probabilities": { "0": 0.349528, "1": 0.453078, "2": 0.162384, "3": 0.03501 } } }, "agent_trace_observability_000083": { "action": { "probabilities": { "continue": 0.171879, "human_review": 0.306396, "observe": 0.48498, "stop": 0.036746 } }, "needs_review": { "probabilities": { "false": 0.605393, "true": 0.394607 } }, "outcome": { "probabilities": { "failure": 0.21463, "harmful": 0.018547, "partial": 0.417242, "success": 0.349581 } }, "risk": { "probabilities": { "0": 0.272451, "1": 0.570427, "2": 0.131085, "3": 0.026037 } }, "urgency": { "probabilities": { "0": 0.324309, "1": 0.451644, "2": 0.192621, "3": 0.031426 } } }, "agent_trace_observability_000084": { "action": { "probabilities": { "continue": 0.55331, "human_review": 0.102314, "observe": 0.329637, "stop": 0.014739 } }, "needs_review": { "probabilities": { "false": 0.856396, "true": 0.143604 } }, "outcome": { "probabilities": { "failure": 0.049614, "harmful": 0.012173, "partial": 0.219773, "success": 0.71844 } }, "risk": { "probabilities": { "0": 0.569708, "1": 0.353941, "2": 0.058449, "3": 0.017902 } }, "urgency": { "probabilities": { "0": 0.640818, "1": 0.283501, "2": 0.065815, "3": 0.009866 } } }, "agent_trace_observability_000085": { "action": { "probabilities": { "continue": 0.201959, "human_review": 0.282569, "observe": 0.48652, "stop": 0.028952 } }, "needs_review": { "probabilities": { "false": 0.636017, "true": 0.363983 } }, "outcome": { "probabilities": { "failure": 0.230732, "harmful": 0.014086, "partial": 0.459889, "success": 0.295293 } }, "risk": { "probabilities": { "0": 0.430229, "1": 0.469639, "2": 0.088507, "3": 0.011625 } }, "urgency": { "probabilities": { "0": 0.386509, "1": 0.464314, "2": 0.135144, "3": 0.014033 } } }, "agent_trace_observability_000086": { "action": { "probabilities": { "continue": 0.025321, "human_review": 0.664665, "observe": 0.116702, "stop": 0.193312 } }, "needs_review": { "probabilities": { "false": 0.107736, "true": 0.892264 } }, "outcome": { "probabilities": { "failure": 0.10438, "harmful": 0.689558, "partial": 0.172371, "success": 0.033691 } }, "risk": { "probabilities": { "0": 0.013875, "1": 0.078212, "2": 0.311624, "3": 0.596289 } }, "urgency": { "probabilities": { "0": 0.026034, "1": 0.128627, "2": 0.409635, "3": 0.435704 } } }, "agent_trace_observability_000087": { "action": { "probabilities": { "continue": 0.265467, "human_review": 0.239117, "observe": 0.470237, "stop": 0.02518 } }, "needs_review": { "probabilities": { "false": 0.676152, "true": 0.323848 } }, "outcome": { "probabilities": { "failure": 0.197515, "harmful": 0.0107, "partial": 0.568827, "success": 0.222958 } }, "risk": { "probabilities": { "0": 0.57375, "1": 0.354598, "2": 0.06347, "3": 0.008182 } }, "urgency": { "probabilities": { "0": 0.437477, "1": 0.422977, "2": 0.124031, "3": 0.015515 } } }, "agent_trace_observability_000088": { "action": { "probabilities": { "continue": 0.023153, "human_review": 0.61991, "observe": 0.094624, "stop": 0.262313 } }, "needs_review": { "probabilities": { "false": 0.091277, "true": 0.908723 } }, "outcome": { "probabilities": { "failure": 0.196549, "harmful": 0.591491, "partial": 0.198987, "success": 0.012973 } }, "risk": { "probabilities": { "0": 0.022881, "1": 0.07358, "2": 0.273006, "3": 0.630533 } }, "urgency": { "probabilities": { "0": 0.027707, "1": 0.121136, "2": 0.329141, "3": 0.522016 } } }, "agent_trace_observability_000089": { "action": { "probabilities": { "continue": 0.158198, "human_review": 0.392185, "observe": 0.389543, "stop": 0.060074 } }, "needs_review": { "probabilities": { "false": 0.485565, "true": 0.514435 } }, "outcome": { "probabilities": { "failure": 0.346017, "harmful": 0.021685, "partial": 0.501911, "success": 0.130387 } }, "risk": { "probabilities": { "0": 0.56035, "1": 0.331565, "2": 0.08647, "3": 0.021615 } }, "urgency": { "probabilities": { "0": 0.262918, "1": 0.458919, "2": 0.222084, "3": 0.05608 } } }, "agent_trace_observability_000090": { "action": { "probabilities": { "continue": 0.584532, "human_review": 0.115369, "observe": 0.275438, "stop": 0.024661 } }, "needs_review": { "probabilities": { "false": 0.819819, "true": 0.180181 } }, "outcome": { "probabilities": { "failure": 0.148496, "harmful": 0.019541, "partial": 0.39045, "success": 0.441514 } }, "risk": { "probabilities": { "0": 0.697443, "1": 0.24901, "2": 0.04539, "3": 0.008157 } }, "urgency": { "probabilities": { "0": 0.599069, "1": 0.319593, "2": 0.065023, "3": 0.016315 } } }, "agent_trace_observability_000091": { "action": { "probabilities": { "continue": 0.027839, "human_review": 0.635844, "observe": 0.120076, "stop": 0.216241 } }, "needs_review": { "probabilities": { "false": 0.127251, "true": 0.872749 } }, "outcome": { "probabilities": { "failure": 0.066449, "harmful": 0.772666, "partial": 0.107821, "success": 0.053064 } }, "risk": { "probabilities": { "0": 0.009478, "1": 0.050009, "2": 0.266559, "3": 0.673954 } }, "urgency": { "probabilities": { "0": 0.030293, "1": 0.100538, "2": 0.325514, "3": 0.543655 } } }, "agent_trace_observability_000092": { "action": { "probabilities": { "continue": 0.176011, "human_review": 0.406631, "observe": 0.354031, "stop": 0.063327 } }, "needs_review": { "probabilities": { "false": 0.52517, "true": 0.47483 } }, "outcome": { "probabilities": { "failure": 0.158039, "harmful": 0.089403, "partial": 0.398404, "success": 0.354155 } }, "risk": { "probabilities": { "0": 0.12822, "1": 0.240665, "2": 0.504652, "3": 0.126463 } }, "urgency": { "probabilities": { "0": 0.28424, "1": 0.393141, "2": 0.23953, "3": 0.083089 } } }, "agent_trace_observability_000093": { "action": { "probabilities": { "continue": 0.179368, "human_review": 0.295343, "observe": 0.494848, "stop": 0.030441 } }, "needs_review": { "probabilities": { "false": 0.620478, "true": 0.379522 } }, "outcome": { "probabilities": { "failure": 0.237111, "harmful": 0.014592, "partial": 0.465576, "success": 0.282721 } }, "risk": { "probabilities": { "0": 0.405259, "1": 0.48684, "2": 0.095396, "3": 0.012505 } }, "urgency": { "probabilities": { "0": 0.378038, "1": 0.462847, "2": 0.14368, "3": 0.015435 } } }, "agent_trace_observability_000094": { "action": { "probabilities": { "continue": 0.163714, "human_review": 0.421755, "observe": 0.363502, "stop": 0.051029 } }, "needs_review": { "probabilities": { "false": 0.487621, "true": 0.512379 } }, "outcome": { "probabilities": { "failure": 0.387379, "harmful": 0.032434, "partial": 0.42202, "success": 0.158167 } }, "risk": { "probabilities": { "0": 0.31848, "1": 0.475066, "2": 0.15801, "3": 0.048444 } }, "urgency": { "probabilities": { "0": 0.22127, "1": 0.494179, "2": 0.233738, "3": 0.050813 } } }, "agent_trace_observability_000095": { "action": { "probabilities": { "continue": 0.334517, "human_review": 0.175175, "observe": 0.462977, "stop": 0.027332 } }, "needs_review": { "probabilities": { "false": 0.789951, "true": 0.210049 } }, "outcome": { "probabilities": { "failure": 0.070629, "harmful": 0.020075, "partial": 0.227302, "success": 0.681994 } }, "risk": { "probabilities": { "0": 0.246864, "1": 0.58209, "2": 0.133879, "3": 0.037167 } }, "urgency": { "probabilities": { "0": 0.480817, "1": 0.363891, "2": 0.127494, "3": 0.027798 } } }, "agent_trace_observability_000096": { "action": { "probabilities": { "continue": 0.018582, "human_review": 0.617607, "observe": 0.078399, "stop": 0.285412 } }, "needs_review": { "probabilities": { "false": 0.078686, "true": 0.921314 } }, "outcome": { "probabilities": { "failure": 0.133677, "harmful": 0.728704, "partial": 0.124295, "success": 0.013324 } }, "risk": { "probabilities": { "0": 0.009408, "1": 0.054211, "2": 0.2723, "3": 0.664081 } }, "urgency": { "probabilities": { "0": 0.018556, "1": 0.098647, "2": 0.349432, "3": 0.533365 } } }, "agent_trace_observability_000097": { "action": { "probabilities": { "continue": 0.125008, "human_review": 0.436083, "observe": 0.385771, "stop": 0.053137 } }, "needs_review": { "probabilities": { "false": 0.458046, "true": 0.541954 } }, "outcome": { "probabilities": { "failure": 0.472242, "harmful": 0.018043, "partial": 0.397807, "success": 0.111908 } }, "risk": { "probabilities": { "0": 0.368109, "1": 0.487965, "2": 0.125543, "3": 0.018383 } }, "urgency": { "probabilities": { "0": 0.237028, "1": 0.51337, "2": 0.218763, "3": 0.030839 } } }, "agent_trace_observability_000098": { "action": { "probabilities": { "continue": 0.159862, "human_review": 0.427817, "observe": 0.34536, "stop": 0.06696 } }, "needs_review": { "probabilities": { "false": 0.442114, "true": 0.557886 } }, "outcome": { "probabilities": { "failure": 0.296943, "harmful": 0.034118, "partial": 0.530019, "success": 0.13892 } }, "risk": { "probabilities": { "0": 0.252094, "1": 0.464537, "2": 0.211944, "3": 0.071425 } }, "urgency": { "probabilities": { "0": 0.130688, "1": 0.483374, "2": 0.298573, "3": 0.087365 } } }, "agent_trace_observability_000099": { "action": { "probabilities": { "continue": 0.090321, "human_review": 0.532099, "observe": 0.294405, "stop": 0.083175 } }, "needs_review": { "probabilities": { "false": 0.341738, "true": 0.658262 } }, "outcome": { "probabilities": { "failure": 0.244837, "harmful": 0.074953, "partial": 0.514902, "success": 0.165308 } }, "risk": { "probabilities": { "0": 0.042815, "1": 0.282482, "2": 0.556006, "3": 0.118697 } }, "urgency": { "probabilities": { "0": 0.106357, "1": 0.421877, "2": 0.371815, "3": 0.099951 } } }, "customer_service_000000": { "action": { "probabilities": { "answer_directly": 0.211695, "close_no_action": 0.018201, "escalate_to_human": 0.3774, "execute_refund": 0.077141, "request_information": 0.315563 } }, "category": { "probabilities": { "account": 0.001865, "billing": 0.003727, "delivery": 0.896973, "refund": 0.089181, "technical": 0.008254 } }, "churn_risk": { "probabilities": { "0": 0.18099, "1": 0.628714, "2": 0.177515, "3": 0.012781 } }, "needs_human": { "probabilities": { "false": 0.484047, "true": 0.515953 } }, "urgency": { "probabilities": { "0": 0.031148, "1": 0.53869, "2": 0.340337, "3": 0.089825 } } }, "customer_service_000001": { "action": { "probabilities": { "answer_directly": 0.269084, "close_no_action": 0.010414, "escalate_to_human": 0.439003, "execute_refund": 0.146466, "request_information": 0.135033 } }, "category": { "probabilities": { "account": 0.010386, "billing": 0.897369, "delivery": 0.003289, "refund": 0.084528, "technical": 0.004428 } }, "churn_risk": { "probabilities": { "0": 0.179499, "1": 0.57649, "2": 0.231645, "3": 0.012366 } }, "needs_human": { "probabilities": { "false": 0.440409, "true": 0.559591 } }, "urgency": { "probabilities": { "0": 0.030955, "1": 0.441256, "2": 0.416034, "3": 0.111755 } } }, "customer_service_000002": { "action": { "probabilities": { "answer_directly": 0.331194, "close_no_action": 0.013775, "escalate_to_human": 0.290222, "execute_refund": 0.042202, "request_information": 0.322606 } }, "category": { "probabilities": { "account": 0.01671, "billing": 0.951776, "delivery": 0.002075, "refund": 0.020847, "technical": 0.008592 } }, "churn_risk": { "probabilities": { "0": 0.203939, "1": 0.533344, "2": 0.227032, "3": 0.035685 } }, "needs_human": { "probabilities": { "false": 0.525654, "true": 0.474346 } }, "urgency": { "probabilities": { "0": 0.050851, "1": 0.461754, "2": 0.3839, "3": 0.103495 } } }, "customer_service_000003": { "action": { "probabilities": { "answer_directly": 0.313271, "close_no_action": 0.014273, "escalate_to_human": 0.480621, "execute_refund": 0.028798, "request_information": 0.163037 } }, "category": { "probabilities": { "account": 0.033987, "billing": 0.939359, "delivery": 0.004781, "refund": 0.006895, "technical": 0.014978 } }, "churn_risk": { "probabilities": { "0": 0.038158, "1": 0.287577, "2": 0.438786, "3": 0.235479 } }, "needs_human": { "probabilities": { "false": 0.384856, "true": 0.615144 } }, "urgency": { "probabilities": { "0": 0.004519, "1": 0.044361, "2": 0.19902, "3": 0.7521 } } }, "customer_service_000004": { "action": { "probabilities": { "answer_directly": 0.799895, "close_no_action": 0.01818, "escalate_to_human": 0.136693, "execute_refund": 0.015636, "request_information": 0.029596 } }, "category": { "probabilities": { "account": 0.065205, "billing": 0.921915, "delivery": 0.001287, "refund": 0.003368, "technical": 0.008225 } }, "churn_risk": { "probabilities": { "0": 0.03162, "1": 0.17306, "2": 0.588121, "3": 0.207199 } }, "needs_human": { "probabilities": { "false": 0.760222, "true": 0.239778 } }, "urgency": { "probabilities": { "0": 0.022447, "1": 0.174505, "2": 0.502873, "3": 0.300175 } } }, "customer_service_000005": { "action": { "probabilities": { "answer_directly": 0.272816, "close_no_action": 0.010153, "escalate_to_human": 0.366226, "execute_refund": 0.070229, "request_information": 0.280577 } }, "category": { "probabilities": { "account": 0.018401, "billing": 0.863066, "delivery": 0.007397, "refund": 0.064135, "technical": 0.047001 } }, "churn_risk": { "probabilities": { "0": 0.030338, "1": 0.245988, "2": 0.466812, "3": 0.256862 } }, "needs_human": { "probabilities": { "false": 0.428187, "true": 0.571813 } }, "urgency": { "probabilities": { "0": 0.004761, "1": 0.038982, "2": 0.159515, "3": 0.796742 } } }, "customer_service_000006": { "action": { "probabilities": { "answer_directly": 0.101331, "close_no_action": 0.011174, "escalate_to_human": 0.57596, "execute_refund": 0.004574, "request_information": 0.306961 } }, "category": { "probabilities": { "account": 0.030314, "billing": 0.021973, "delivery": 0.018392, "refund": 0.006637, "technical": 0.922684 } }, "churn_risk": { "probabilities": { "0": 0.05541, "1": 0.435622, "2": 0.483131, "3": 0.025836 } }, "needs_human": { "probabilities": { "false": 0.27033, "true": 0.72967 } }, "urgency": { "probabilities": { "0": 0.023827, "1": 0.238162, "2": 0.538714, "3": 0.199297 } } }, "customer_service_000007": { "action": { "probabilities": { "answer_directly": 0.201805, "close_no_action": 0.017512, "escalate_to_human": 0.359697, "execute_refund": 0.239273, "request_information": 0.181713 } }, "category": { "probabilities": { "account": 0.05033, "billing": 0.09811, "delivery": 0.017603, "refund": 0.823044, "technical": 0.010913 } }, "churn_risk": { "probabilities": { "0": 0.003132, "1": 0.037726, "2": 0.132316, "3": 0.826826 } }, "needs_human": { "probabilities": { "false": 0.377915, "true": 0.622085 } }, "urgency": { "probabilities": { "0": 0.004326, "1": 0.063489, "2": 0.49036, "3": 0.441825 } } }, "customer_service_000008": { "action": { "probabilities": { "answer_directly": 0.17694, "close_no_action": 0.007719, "escalate_to_human": 0.472439, "execute_refund": 0.009168, "request_information": 0.333734 } }, "category": { "probabilities": { "account": 0.294635, "billing": 0.607161, "delivery": 0.001399, "refund": 0.00305, "technical": 0.093755 } }, "churn_risk": { "probabilities": { "0": 0.046422, "1": 0.407893, "2": 0.501931, "3": 0.043754 } }, "needs_human": { "probabilities": { "false": 0.352282, "true": 0.647718 } }, "urgency": { "probabilities": { "0": 0.012935, "1": 0.147523, "2": 0.609982, "3": 0.22956 } } }, "customer_service_000009": { "action": { "probabilities": { "answer_directly": 0.079448, "close_no_action": 0.012216, "escalate_to_human": 0.45177, "execute_refund": 0.017316, "request_information": 0.43925 } }, "category": { "probabilities": { "account": 0.091562, "billing": 0.23167, "delivery": 0.001824, "refund": 0.004879, "technical": 0.670065 } }, "churn_risk": { "probabilities": { "0": 0.008335, "1": 0.129951, "2": 0.456095, "3": 0.40562 } }, "needs_human": { "probabilities": { "false": 0.279505, "true": 0.720495 } }, "urgency": { "probabilities": { "0": 0.015005, "1": 0.149152, "2": 0.547166, "3": 0.288677 } } }, "customer_service_000010": { "action": { "probabilities": { "answer_directly": 0.079392, "close_no_action": 0.012146, "escalate_to_human": 0.57152, "execute_refund": 0.005665, "request_information": 0.331277 } }, "category": { "probabilities": { "account": 0.045287, "billing": 0.022623, "delivery": 0.018776, "refund": 0.007308, "technical": 0.906006 } }, "churn_risk": { "probabilities": { "0": 0.020524, "1": 0.213239, "2": 0.61473, "3": 0.151507 } }, "needs_human": { "probabilities": { "false": 0.243554, "true": 0.756446 } }, "urgency": { "probabilities": { "0": 0.022627, "1": 0.245561, "2": 0.612562, "3": 0.11925 } } }, "customer_service_000011": { "action": { "probabilities": { "answer_directly": 0.344021, "close_no_action": 0.012384, "escalate_to_human": 0.398865, "execute_refund": 0.024585, "request_information": 0.220145 } }, "category": { "probabilities": { "account": 0.171497, "billing": 0.780799, "delivery": 0.002945, "refund": 0.010219, "technical": 0.03454 } }, "churn_risk": { "probabilities": { "0": 0.044106, "1": 0.328549, "2": 0.502841, "3": 0.124504 } }, "needs_human": { "probabilities": { "false": 0.447654, "true": 0.552346 } }, "urgency": { "probabilities": { "0": 0.006617, "1": 0.034036, "2": 0.119846, "3": 0.839501 } } }, "customer_service_000012": { "action": { "probabilities": { "answer_directly": 0.148505, "close_no_action": 0.012352, "escalate_to_human": 0.574678, "execute_refund": 0.114041, "request_information": 0.150424 } }, "category": { "probabilities": { "account": 0.019971, "billing": 0.900741, "delivery": 0.003526, "refund": 0.058147, "technical": 0.017615 } }, "churn_risk": { "probabilities": { "0": 0.02673, "1": 0.229078, "2": 0.672112, "3": 0.07208 } }, "needs_human": { "probabilities": { "false": 0.249615, "true": 0.750385 } }, "urgency": { "probabilities": { "0": 0.032879, "1": 0.322331, "2": 0.551714, "3": 0.093076 } } }, "customer_service_000013": { "action": { "probabilities": { "answer_directly": 0.283909, "close_no_action": 0.026073, "escalate_to_human": 0.411547, "execute_refund": 0.012649, "request_information": 0.265822 } }, "category": { "probabilities": { "account": 0.890203, "billing": 0.086034, "delivery": 0.002217, "refund": 0.007408, "technical": 0.014138 } }, "churn_risk": { "probabilities": { "0": 0.162752, "1": 0.385272, "2": 0.310212, "3": 0.141764 } }, "needs_human": { "probabilities": { "false": 0.436505, "true": 0.563495 } }, "urgency": { "probabilities": { "0": 0.124368, "1": 0.648497, "2": 0.198569, "3": 0.028566 } } }, "customer_service_000014": { "action": { "probabilities": { "answer_directly": 0.247193, "close_no_action": 0.011654, "escalate_to_human": 0.57109, "execute_refund": 0.003446, "request_information": 0.166617 } }, "category": { "probabilities": { "account": 0.194543, "billing": 0.03334, "delivery": 0.010192, "refund": 0.002375, "technical": 0.75955 } }, "churn_risk": { "probabilities": { "0": 0.055894, "1": 0.454723, "2": 0.421954, "3": 0.067429 } }, "needs_human": { "probabilities": { "false": 0.388229, "true": 0.611771 } }, "urgency": { "probabilities": { "0": 0.005031, "1": 0.066953, "2": 0.282504, "3": 0.645512 } } }, "customer_service_000015": { "action": { "probabilities": { "answer_directly": 0.16349, "close_no_action": 0.011563, "escalate_to_human": 0.582406, "execute_refund": 0.007342, "request_information": 0.235199 } }, "category": { "probabilities": { "account": 0.032842, "billing": 0.014368, "delivery": 0.029389, "refund": 0.004022, "technical": 0.919379 } }, "churn_risk": { "probabilities": { "0": 0.074737, "1": 0.444596, "2": 0.399668, "3": 0.080999 } }, "needs_human": { "probabilities": { "false": 0.273923, "true": 0.726077 } }, "urgency": { "probabilities": { "0": 0.007966, "1": 0.074026, "2": 0.299082, "3": 0.618926 } } }, "customer_service_000016": { "action": { "probabilities": { "answer_directly": 0.564687, "close_no_action": 0.013626, "escalate_to_human": 0.211489, "execute_refund": 0.012092, "request_information": 0.198106 } }, "category": { "probabilities": { "account": 0.003782, "billing": 0.009442, "delivery": 0.97966, "refund": 0.005365, "technical": 0.001751 } }, "churn_risk": { "probabilities": { "0": 0.210241, "1": 0.601754, "2": 0.174902, "3": 0.013103 } }, "needs_human": { "probabilities": { "false": 0.670318, "true": 0.329682 } }, "urgency": { "probabilities": { "0": 0.009443, "1": 0.168511, "2": 0.595899, "3": 0.226147 } } }, "customer_service_000017": { "action": { "probabilities": { "answer_directly": 0.075486, "close_no_action": 0.015716, "escalate_to_human": 0.742222, "execute_refund": 0.015448, "request_information": 0.151128 } }, "category": { "probabilities": { "account": 0.077626, "billing": 0.020411, "delivery": 0.038921, "refund": 0.008702, "technical": 0.85434 } }, "churn_risk": { "probabilities": { "0": 0.004863, "1": 0.055578, "2": 0.239083, "3": 0.700476 } }, "needs_human": { "probabilities": { "false": 0.130222, "true": 0.869778 } }, "urgency": { "probabilities": { "0": 0.006233, "1": 0.058474, "2": 0.232958, "3": 0.702335 } } }, "customer_service_000018": { "action": { "probabilities": { "answer_directly": 0.238209, "close_no_action": 0.01478, "escalate_to_human": 0.472968, "execute_refund": 0.146755, "request_information": 0.127289 } }, "category": { "probabilities": { "account": 0.04352, "billing": 0.851958, "delivery": 0.017451, "refund": 0.051152, "technical": 0.035919 } }, "churn_risk": { "probabilities": { "0": 0.012599, "1": 0.085328, "2": 0.313881, "3": 0.588192 } }, "needs_human": { "probabilities": { "false": 0.308947, "true": 0.691053 } }, "urgency": { "probabilities": { "0": 0.005542, "1": 0.046958, "2": 0.123045, "3": 0.824455 } } }, "customer_service_000019": { "action": { "probabilities": { "answer_directly": 0.129906, "close_no_action": 0.011166, "escalate_to_human": 0.616831, "execute_refund": 0.074487, "request_information": 0.16761 } }, "category": { "probabilities": { "account": 0.026086, "billing": 0.911568, "delivery": 0.004134, "refund": 0.04218, "technical": 0.016032 } }, "churn_risk": { "probabilities": { "0": 0.009972, "1": 0.136157, "2": 0.538425, "3": 0.315446 } }, "needs_human": { "probabilities": { "false": 0.230682, "true": 0.769318 } }, "urgency": { "probabilities": { "0": 0.013931, "1": 0.189628, "2": 0.598484, "3": 0.197957 } } }, "customer_service_000020": { "action": { "probabilities": { "answer_directly": 0.105618, "close_no_action": 0.015114, "escalate_to_human": 0.621775, "execute_refund": 0.016672, "request_information": 0.240821 } }, "category": { "probabilities": { "account": 0.082646, "billing": 0.021718, "delivery": 0.009905, "refund": 0.008636, "technical": 0.877095 } }, "churn_risk": { "probabilities": { "0": 0.003409, "1": 0.066761, "2": 0.323344, "3": 0.606486 } }, "needs_human": { "probabilities": { "false": 0.225999, "true": 0.774001 } }, "urgency": { "probabilities": { "0": 0.006753, "1": 0.106636, "2": 0.688106, "3": 0.198505 } } }, "customer_service_000021": { "action": { "probabilities": { "answer_directly": 0.180854, "close_no_action": 0.012851, "escalate_to_human": 0.345453, "execute_refund": 0.004714, "request_information": 0.456128 } }, "category": { "probabilities": { "account": 0.035488, "billing": 0.020562, "delivery": 0.027377, "refund": 0.008216, "technical": 0.908357 } }, "churn_risk": { "probabilities": { "0": 0.059452, "1": 0.420368, "2": 0.422593, "3": 0.097587 } }, "needs_human": { "probabilities": { "false": 0.487902, "true": 0.512098 } }, "urgency": { "probabilities": { "0": 0.032765, "1": 0.46551, "2": 0.430636, "3": 0.071089 } } }, "customer_service_000022": { "action": { "probabilities": { "answer_directly": 0.254047, "close_no_action": 0.013937, "escalate_to_human": 0.544222, "execute_refund": 0.005918, "request_information": 0.181876 } }, "category": { "probabilities": { "account": 0.870966, "billing": 0.029683, "delivery": 0.006591, "refund": 0.002372, "technical": 0.090388 } }, "churn_risk": { "probabilities": { "0": 0.030942, "1": 0.27339, "2": 0.482647, "3": 0.213021 } }, "needs_human": { "probabilities": { "false": 0.346862, "true": 0.653138 } }, "urgency": { "probabilities": { "0": 0.005467, "1": 0.040068, "2": 0.130325, "3": 0.82414 } } }, "customer_service_000023": { "action": { "probabilities": { "answer_directly": 0.266982, "close_no_action": 0.010874, "escalate_to_human": 0.402223, "execute_refund": 0.095826, "request_information": 0.224095 } }, "category": { "probabilities": { "account": 0.006857, "billing": 0.938917, "delivery": 0.003115, "refund": 0.046462, "technical": 0.004649 } }, "churn_risk": { "probabilities": { "0": 0.152848, "1": 0.525408, "2": 0.296751, "3": 0.024993 } }, "needs_human": { "probabilities": { "false": 0.474605, "true": 0.525395 } }, "urgency": { "probabilities": { "0": 0.026748, "1": 0.400419, "2": 0.429355, "3": 0.143478 } } }, "customer_service_000024": { "action": { "probabilities": { "answer_directly": 0.399742, "close_no_action": 0.013228, "escalate_to_human": 0.49178, "execute_refund": 0.024514, "request_information": 0.070735 } }, "category": { "probabilities": { "account": 0.054281, "billing": 0.897246, "delivery": 0.005426, "refund": 0.006105, "technical": 0.036942 } }, "churn_risk": { "probabilities": { "0": 0.05918, "1": 0.305969, "2": 0.540906, "3": 0.093945 } }, "needs_human": { "probabilities": { "false": 0.410333, "true": 0.589667 } }, "urgency": { "probabilities": { "0": 0.00492, "1": 0.028081, "2": 0.102587, "3": 0.864412 } } }, "customer_service_000025": { "action": { "probabilities": { "answer_directly": 0.166064, "close_no_action": 0.014257, "escalate_to_human": 0.485377, "execute_refund": 0.00655, "request_information": 0.327753 } }, "category": { "probabilities": { "account": 0.76217, "billing": 0.191332, "delivery": 0.000772, "refund": 0.003407, "technical": 0.042319 } }, "churn_risk": { "probabilities": { "0": 0.010146, "1": 0.129432, "2": 0.641845, "3": 0.218577 } }, "needs_human": { "probabilities": { "false": 0.290751, "true": 0.709249 } }, "urgency": { "probabilities": { "0": 0.054217, "1": 0.347819, "2": 0.528118, "3": 0.069846 } } }, "customer_service_000026": { "action": { "probabilities": { "answer_directly": 0.118856, "close_no_action": 0.011985, "escalate_to_human": 0.615269, "execute_refund": 0.150192, "request_information": 0.103698 } }, "category": { "probabilities": { "account": 0.02741, "billing": 0.818945, "delivery": 0.004713, "refund": 0.132479, "technical": 0.016453 } }, "churn_risk": { "probabilities": { "0": 0.018262, "1": 0.181827, "2": 0.684533, "3": 0.115378 } }, "needs_human": { "probabilities": { "false": 0.209175, "true": 0.790825 } }, "urgency": { "probabilities": { "0": 0.018421, "1": 0.183628, "2": 0.598113, "3": 0.199838 } } }, "customer_service_000027": { "action": { "probabilities": { "answer_directly": 0.103897, "close_no_action": 0.008165, "escalate_to_human": 0.754415, "execute_refund": 0.007456, "request_information": 0.126067 } }, "category": { "probabilities": { "account": 0.649113, "billing": 0.303996, "delivery": 0.000491, "refund": 0.002291, "technical": 0.044109 } }, "churn_risk": { "probabilities": { "0": 0.005575, "1": 0.090704, "2": 0.635646, "3": 0.268075 } }, "needs_human": { "probabilities": { "false": 0.141405, "true": 0.858595 } }, "urgency": { "probabilities": { "0": 0.021179, "1": 0.187179, "2": 0.654252, "3": 0.13739 } } }, "customer_service_000028": { "action": { "probabilities": { "answer_directly": 0.173626, "close_no_action": 0.012242, "escalate_to_human": 0.346331, "execute_refund": 0.031606, "request_information": 0.436195 } }, "category": { "probabilities": { "account": 0.001351, "billing": 0.005071, "delivery": 0.966505, "refund": 0.021683, "technical": 0.00539 } }, "churn_risk": { "probabilities": { "0": 0.075764, "1": 0.363364, "2": 0.464476, "3": 0.096396 } }, "needs_human": { "probabilities": { "false": 0.445342, "true": 0.554658 } }, "urgency": { "probabilities": { "0": 0.018223, "1": 0.287329, "2": 0.448924, "3": 0.245525 } } }, "customer_service_000029": { "action": { "probabilities": { "answer_directly": 0.555323, "close_no_action": 0.016873, "escalate_to_human": 0.121234, "execute_refund": 0.167263, "request_information": 0.139307 } }, "category": { "probabilities": { "account": 0.073873, "billing": 0.723873, "delivery": 0.002392, "refund": 0.193058, "technical": 0.006804 } }, "churn_risk": { "probabilities": { "0": 0.067461, "1": 0.178997, "2": 0.173326, "3": 0.580216 } }, "needs_human": { "probabilities": { "false": 0.767533, "true": 0.232467 } }, "urgency": { "probabilities": { "0": 0.009102, "1": 0.131734, "2": 0.586852, "3": 0.272312 } } }, "customer_service_000030": { "action": { "probabilities": { "answer_directly": 0.180127, "close_no_action": 0.010003, "escalate_to_human": 0.251045, "execute_refund": 0.022068, "request_information": 0.536757 } }, "category": { "probabilities": { "account": 0.002131, "billing": 0.010945, "delivery": 0.81217, "refund": 0.080164, "technical": 0.09459 } }, "churn_risk": { "probabilities": { "0": 0.202697, "1": 0.661759, "2": 0.131306, "3": 0.004238 } }, "needs_human": { "probabilities": { "false": 0.59861, "true": 0.40139 } }, "urgency": { "probabilities": { "0": 0.046754, "1": 0.675622, "2": 0.213519, "3": 0.064105 } } }, "customer_service_000031": { "action": { "probabilities": { "answer_directly": 0.103238, "close_no_action": 0.014797, "escalate_to_human": 0.66713, "execute_refund": 0.005888, "request_information": 0.208947 } }, "category": { "probabilities": { "account": 0.052261, "billing": 0.020697, "delivery": 0.024135, "refund": 0.007273, "technical": 0.895634 } }, "churn_risk": { "probabilities": { "0": 0.041971, "1": 0.277743, "2": 0.470515, "3": 0.209771 } }, "needs_human": { "probabilities": { "false": 0.232764, "true": 0.767236 } }, "urgency": { "probabilities": { "0": 0.041473, "1": 0.496507, "2": 0.381255, "3": 0.080765 } } }, "customer_service_000032": { "action": { "probabilities": { "answer_directly": 0.139404, "close_no_action": 0.008718, "escalate_to_human": 0.530166, "execute_refund": 0.132448, "request_information": 0.189264 } }, "category": { "probabilities": { "account": 0.014383, "billing": 0.106063, "delivery": 0.004569, "refund": 0.870445, "technical": 0.00454 } }, "churn_risk": { "probabilities": { "0": 0.019243, "1": 0.24404, "2": 0.571405, "3": 0.165312 } }, "needs_human": { "probabilities": { "false": 0.284922, "true": 0.715078 } }, "urgency": { "probabilities": { "0": 0.010172, "1": 0.118138, "2": 0.695023, "3": 0.176667 } } }, "customer_service_000033": { "action": { "probabilities": { "answer_directly": 0.135929, "close_no_action": 0.00813, "escalate_to_human": 0.19466, "execute_refund": 0.023336, "request_information": 0.637945 } }, "category": { "probabilities": { "account": 0.007372, "billing": 0.018985, "delivery": 0.304683, "refund": 0.443234, "technical": 0.225726 } }, "churn_risk": { "probabilities": { "0": 0.13332, "1": 0.669474, "2": 0.187144, "3": 0.010062 } }, "needs_human": { "probabilities": { "false": 0.606434, "true": 0.393566 } }, "urgency": { "probabilities": { "0": 0.044028, "1": 0.553581, "2": 0.325569, "3": 0.076822 } } }, "customer_service_000034": { "action": { "probabilities": { "answer_directly": 0.094282, "close_no_action": 0.017443, "escalate_to_human": 0.53677, "execute_refund": 0.029058, "request_information": 0.322447 } }, "category": { "probabilities": { "account": 0.006517, "billing": 0.007427, "delivery": 0.783643, "refund": 0.022721, "technical": 0.179692 } }, "churn_risk": { "probabilities": { "0": 0.026421, "1": 0.279618, "2": 0.41764, "3": 0.276321 } }, "needs_human": { "probabilities": { "false": 0.257634, "true": 0.742366 } }, "urgency": { "probabilities": { "0": 0.005953, "1": 0.101405, "2": 0.476803, "3": 0.415839 } } }, "customer_service_000035": { "action": { "probabilities": { "answer_directly": 0.174589, "close_no_action": 0.010455, "escalate_to_human": 0.515354, "execute_refund": 0.068724, "request_information": 0.230878 } }, "category": { "probabilities": { "account": 0.01933, "billing": 0.907138, "delivery": 0.006194, "refund": 0.038445, "technical": 0.028893 } }, "churn_risk": { "probabilities": { "0": 0.029083, "1": 0.217445, "2": 0.508052, "3": 0.24542 } }, "needs_human": { "probabilities": { "false": 0.313378, "true": 0.686622 } }, "urgency": { "probabilities": { "0": 0.007966, "1": 0.104253, "2": 0.459228, "3": 0.428553 } } }, "customer_service_000036": { "action": { "probabilities": { "answer_directly": 0.722798, "close_no_action": 0.031728, "escalate_to_human": 0.13814, "execute_refund": 0.053484, "request_information": 0.05385 } }, "category": { "probabilities": { "account": 0.691009, "billing": 0.185536, "delivery": 0.001487, "refund": 0.111473, "technical": 0.010495 } }, "churn_risk": { "probabilities": { "0": 0.074691, "1": 0.092226, "2": 0.071874, "3": 0.761209 } }, "needs_human": { "probabilities": { "false": 0.743706, "true": 0.256294 } }, "urgency": { "probabilities": { "0": 0.078645, "1": 0.666771, "2": 0.215007, "3": 0.039577 } } }, "customer_service_000037": { "action": { "probabilities": { "answer_directly": 0.092761, "close_no_action": 0.02143, "escalate_to_human": 0.622106, "execute_refund": 0.068385, "request_information": 0.195318 } }, "category": { "probabilities": { "account": 0.010921, "billing": 0.018084, "delivery": 0.899244, "refund": 0.066851, "technical": 0.0049 } }, "churn_risk": { "probabilities": { "0": 0.032082, "1": 0.246202, "2": 0.639847, "3": 0.081869 } }, "needs_human": { "probabilities": { "false": 0.194383, "true": 0.805617 } }, "urgency": { "probabilities": { "0": 0.028653, "1": 0.325021, "2": 0.585318, "3": 0.061008 } } }, "customer_service_000038": { "action": { "probabilities": { "answer_directly": 0.338396, "close_no_action": 0.009065, "escalate_to_human": 0.28377, "execute_refund": 0.000905, "request_information": 0.367864 } }, "category": { "probabilities": { "account": 0.859701, "billing": 0.020684, "delivery": 0.001834, "refund": 0.001136, "technical": 0.116645 } }, "churn_risk": { "probabilities": { "0": 0.141778, "1": 0.596206, "2": 0.23519, "3": 0.026826 } }, "needs_human": { "probabilities": { "false": 0.590455, "true": 0.409545 } }, "urgency": { "probabilities": { "0": 0.040043, "1": 0.534757, "2": 0.318455, "3": 0.106745 } } }, "customer_service_000039": { "action": { "probabilities": { "answer_directly": 0.173868, "close_no_action": 0.011753, "escalate_to_human": 0.185053, "execute_refund": 0.001953, "request_information": 0.627373 } }, "category": { "probabilities": { "account": 0.027449, "billing": 0.018469, "delivery": 0.018679, "refund": 0.006091, "technical": 0.929312 } }, "churn_risk": { "probabilities": { "0": 0.274724, "1": 0.579572, "2": 0.131944, "3": 0.01376 } }, "needs_human": { "probabilities": { "false": 0.629742, "true": 0.370258 } }, "urgency": { "probabilities": { "0": 0.087104, "1": 0.682759, "2": 0.194187, "3": 0.03595 } } }, "customer_service_000040": { "action": { "probabilities": { "answer_directly": 0.192975, "close_no_action": 0.016634, "escalate_to_human": 0.690215, "execute_refund": 0.024262, "request_information": 0.075914 } }, "category": { "probabilities": { "account": 0.089849, "billing": 0.808134, "delivery": 0.011054, "refund": 0.009911, "technical": 0.081052 } }, "churn_risk": { "probabilities": { "0": 0.021967, "1": 0.175629, "2": 0.584178, "3": 0.218226 } }, "needs_human": { "probabilities": { "false": 0.252343, "true": 0.747657 } }, "urgency": { "probabilities": { "0": 0.003997, "1": 0.026902, "2": 0.084095, "3": 0.885006 } } }, "customer_service_000041": { "action": { "probabilities": { "answer_directly": 0.054517, "close_no_action": 0.014909, "escalate_to_human": 0.7528, "execute_refund": 0.01809, "request_information": 0.159684 } }, "category": { "probabilities": { "account": 0.052743, "billing": 0.035668, "delivery": 0.033924, "refund": 0.022721, "technical": 0.854944 } }, "churn_risk": { "probabilities": { "0": 0.003486, "1": 0.046352, "2": 0.218322, "3": 0.73184 } }, "needs_human": { "probabilities": { "false": 0.143625, "true": 0.856375 } }, "urgency": { "probabilities": { "0": 0.00395, "1": 0.053726, "2": 0.250644, "3": 0.69168 } } }, "customer_service_000042": { "action": { "probabilities": { "answer_directly": 0.13082, "close_no_action": 0.010853, "escalate_to_human": 0.442546, "execute_refund": 0.113525, "request_information": 0.302256 } }, "category": { "probabilities": { "account": 0.042317, "billing": 0.853895, "delivery": 0.003808, "refund": 0.069141, "technical": 0.030839 } }, "churn_risk": { "probabilities": { "0": 0.009956, "1": 0.126073, "2": 0.371271, "3": 0.4927 } }, "needs_human": { "probabilities": { "false": 0.292046, "true": 0.707954 } }, "urgency": { "probabilities": { "0": 0.008158, "1": 0.081154, "2": 0.424447, "3": 0.486242 } } }, "customer_service_000043": { "action": { "probabilities": { "answer_directly": 0.204232, "close_no_action": 0.012331, "escalate_to_human": 0.628424, "execute_refund": 0.049683, "request_information": 0.10533 } }, "category": { "probabilities": { "account": 0.175861, "billing": 0.780361, "delivery": 0.002067, "refund": 0.014456, "technical": 0.027255 } }, "churn_risk": { "probabilities": { "0": 0.018763, "1": 0.168149, "2": 0.64195, "3": 0.171138 } }, "needs_human": { "probabilities": { "false": 0.278255, "true": 0.721745 } }, "urgency": { "probabilities": { "0": 0.021746, "1": 0.208588, "2": 0.496852, "3": 0.272814 } } }, "customer_service_000044": { "action": { "probabilities": { "answer_directly": 0.196804, "close_no_action": 0.015469, "escalate_to_human": 0.420666, "execute_refund": 0.008626, "request_information": 0.358434 } }, "category": { "probabilities": { "account": 0.018307, "billing": 0.014793, "delivery": 0.221741, "refund": 0.00985, "technical": 0.735309 } }, "churn_risk": { "probabilities": { "0": 0.154218, "1": 0.555485, "2": 0.26585, "3": 0.024447 } }, "needs_human": { "probabilities": { "false": 0.434672, "true": 0.565328 } }, "urgency": { "probabilities": { "0": 0.010062, "1": 0.155044, "2": 0.59714, "3": 0.237754 } } }, "customer_service_000045": { "action": { "probabilities": { "answer_directly": 0.78345, "close_no_action": 0.015305, "escalate_to_human": 0.096951, "execute_refund": 0.016827, "request_information": 0.087467 } }, "category": { "probabilities": { "account": 0.011574, "billing": 0.974518, "delivery": 0.003723, "refund": 0.00487, "technical": 0.005315 } }, "churn_risk": { "probabilities": { "0": 0.636688, "1": 0.288696, "2": 0.069732, "3": 0.004884 } }, "needs_human": { "probabilities": { "false": 0.842373, "true": 0.157627 } }, "urgency": { "probabilities": { "0": 0.016023, "1": 0.091848, "2": 0.198972, "3": 0.693157 } } }, "customer_service_000046": { "action": { "probabilities": { "answer_directly": 0.26298, "close_no_action": 0.010103, "escalate_to_human": 0.473558, "execute_refund": 0.103469, "request_information": 0.14989 } }, "category": { "probabilities": { "account": 0.011733, "billing": 0.935992, "delivery": 0.004671, "refund": 0.041013, "technical": 0.006591 } }, "churn_risk": { "probabilities": { "0": 0.081757, "1": 0.392898, "2": 0.427705, "3": 0.09764 } }, "needs_human": { "probabilities": { "false": 0.381844, "true": 0.618156 } }, "urgency": { "probabilities": { "0": 0.00714, "1": 0.134696, "2": 0.561005, "3": 0.297159 } } }, "customer_service_000047": { "action": { "probabilities": { "answer_directly": 0.22461, "close_no_action": 0.010242, "escalate_to_human": 0.315835, "execute_refund": 0.009087, "request_information": 0.440226 } }, "category": { "probabilities": { "account": 0.330124, "billing": 0.476377, "delivery": 0.003126, "refund": 0.004733, "technical": 0.185639 } }, "churn_risk": { "probabilities": { "0": 0.052065, "1": 0.317916, "2": 0.433527, "3": 0.196492 } }, "needs_human": { "probabilities": { "false": 0.490402, "true": 0.509598 } }, "urgency": { "probabilities": { "0": 0.009802, "1": 0.107471, "2": 0.316246, "3": 0.566481 } } }, "customer_service_000048": { "action": { "probabilities": { "answer_directly": 0.170981, "close_no_action": 0.018789, "escalate_to_human": 0.558716, "execute_refund": 0.033193, "request_information": 0.218321 } }, "category": { "probabilities": { "account": 0.009378, "billing": 0.018169, "delivery": 0.940303, "refund": 0.026125, "technical": 0.006025 } }, "churn_risk": { "probabilities": { "0": 0.029117, "1": 0.218518, "2": 0.57802, "3": 0.174345 } }, "needs_human": { "probabilities": { "false": 0.299525, "true": 0.700475 } }, "urgency": { "probabilities": { "0": 0.013293, "1": 0.175404, "2": 0.613867, "3": 0.197436 } } }, "customer_service_000049": { "action": { "probabilities": { "answer_directly": 0.235296, "close_no_action": 0.013892, "escalate_to_human": 0.554009, "execute_refund": 0.027267, "request_information": 0.169536 } }, "category": { "probabilities": { "account": 0.052124, "billing": 0.818978, "delivery": 0.004137, "refund": 0.006475, "technical": 0.118286 } }, "churn_risk": { "probabilities": { "0": 0.03554, "1": 0.251286, "2": 0.607796, "3": 0.105378 } }, "needs_human": { "probabilities": { "false": 0.331202, "true": 0.668798 } }, "urgency": { "probabilities": { "0": 0.007995, "1": 0.033514, "2": 0.087746, "3": 0.870745 } } }, "customer_service_000050": { "action": { "probabilities": { "answer_directly": 0.076936, "close_no_action": 0.011936, "escalate_to_human": 0.711491, "execute_refund": 0.004579, "request_information": 0.195058 } }, "category": { "probabilities": { "account": 0.67737, "billing": 0.013156, "delivery": 0.00144, "refund": 0.001488, "technical": 0.306546 } }, "churn_risk": { "probabilities": { "0": 0.001812, "1": 0.061947, "2": 0.448006, "3": 0.488235 } }, "needs_human": { "probabilities": { "false": 0.176912, "true": 0.823088 } }, "urgency": { "probabilities": { "0": 0.00898, "1": 0.096131, "2": 0.458699, "3": 0.436191 } } }, "customer_service_000051": { "action": { "probabilities": { "answer_directly": 0.102999, "close_no_action": 0.016051, "escalate_to_human": 0.719127, "execute_refund": 0.024519, "request_information": 0.137304 } }, "category": { "probabilities": { "account": 0.034966, "billing": 0.009728, "delivery": 0.570713, "refund": 0.022611, "technical": 0.361982 } }, "churn_risk": { "probabilities": { "0": 0.00358, "1": 0.054169, "2": 0.308737, "3": 0.633514 } }, "needs_human": { "probabilities": { "false": 0.146413, "true": 0.853587 } }, "urgency": { "probabilities": { "0": 0.004441, "1": 0.038126, "2": 0.15684, "3": 0.800593 } } }, "customer_service_000052": { "action": { "probabilities": { "answer_directly": 0.367749, "close_no_action": 0.018819, "escalate_to_human": 0.503326, "execute_refund": 0.07492, "request_information": 0.035186 } }, "category": { "probabilities": { "account": 0.33206, "billing": 0.622843, "delivery": 0.00289, "refund": 0.022812, "technical": 0.019395 } }, "churn_risk": { "probabilities": { "0": 0.003789, "1": 0.043926, "2": 0.305178, "3": 0.647107 } }, "needs_human": { "probabilities": { "false": 0.323426, "true": 0.676574 } }, "urgency": { "probabilities": { "0": 0.002659, "1": 0.018695, "2": 0.089197, "3": 0.889449 } } }, "customer_service_000053": { "action": { "probabilities": { "answer_directly": 0.22562, "close_no_action": 0.013426, "escalate_to_human": 0.453471, "execute_refund": 0.035702, "request_information": 0.27178 } }, "category": { "probabilities": { "account": 0.036792, "billing": 0.848212, "delivery": 0.014045, "refund": 0.0219, "technical": 0.079051 } }, "churn_risk": { "probabilities": { "0": 0.032917, "1": 0.206061, "2": 0.51732, "3": 0.243702 } }, "needs_human": { "probabilities": { "false": 0.408772, "true": 0.591228 } }, "urgency": { "probabilities": { "0": 0.013802, "1": 0.133751, "2": 0.330183, "3": 0.522264 } } }, "customer_service_000054": { "action": { "probabilities": { "answer_directly": 0.160942, "close_no_action": 0.013599, "escalate_to_human": 0.705566, "execute_refund": 0.004752, "request_information": 0.115141 } }, "category": { "probabilities": { "account": 0.479293, "billing": 0.074477, "delivery": 0.045955, "refund": 0.0095, "technical": 0.390774 } }, "churn_risk": { "probabilities": { "0": 0.003727, "1": 0.056007, "2": 0.26222, "3": 0.678046 } }, "needs_human": { "probabilities": { "false": 0.211381, "true": 0.788619 } }, "urgency": { "probabilities": { "0": 0.001775, "1": 0.019438, "2": 0.076404, "3": 0.902383 } } }, "customer_service_000055": { "action": { "probabilities": { "answer_directly": 0.106669, "close_no_action": 0.012194, "escalate_to_human": 0.528891, "execute_refund": 0.257697, "request_information": 0.094549 } }, "category": { "probabilities": { "account": 0.021097, "billing": 0.77792, "delivery": 0.012339, "refund": 0.173349, "technical": 0.015295 } }, "churn_risk": { "probabilities": { "0": 0.009896, "1": 0.077235, "2": 0.388069, "3": 0.5248 } }, "needs_human": { "probabilities": { "false": 0.229019, "true": 0.770981 } }, "urgency": { "probabilities": { "0": 0.008086, "1": 0.146641, "2": 0.723044, "3": 0.122229 } } }, "customer_service_000056": { "action": { "probabilities": { "answer_directly": 0.632679, "close_no_action": 0.03086, "escalate_to_human": 0.255979, "execute_refund": 0.021299, "request_information": 0.059183 } }, "category": { "probabilities": { "account": 0.891311, "billing": 0.073305, "delivery": 0.005401, "refund": 0.01429, "technical": 0.015693 } }, "churn_risk": { "probabilities": { "0": 0.003761, "1": 0.029812, "2": 0.127046, "3": 0.839381 } }, "needs_human": { "probabilities": { "false": 0.579193, "true": 0.420807 } }, "urgency": { "probabilities": { "0": 0.006234, "1": 0.052881, "2": 0.251437, "3": 0.689448 } } }, "customer_service_000057": { "action": { "probabilities": { "answer_directly": 0.79289, "close_no_action": 0.024607, "escalate_to_human": 0.123928, "execute_refund": 0.019556, "request_information": 0.039019 } }, "category": { "probabilities": { "account": 0.050344, "billing": 0.925964, "delivery": 0.00875, "refund": 0.004069, "technical": 0.010873 } }, "churn_risk": { "probabilities": { "0": 0.010219, "1": 0.057125, "2": 0.1868, "3": 0.745856 } }, "needs_human": { "probabilities": { "false": 0.743699, "true": 0.256301 } }, "urgency": { "probabilities": { "0": 0.00275, "1": 0.019657, "2": 0.104019, "3": 0.873574 } } }, "customer_service_000058": { "action": { "probabilities": { "answer_directly": 0.085941, "close_no_action": 0.012676, "escalate_to_human": 0.295143, "execute_refund": 0.011563, "request_information": 0.594677 } }, "category": { "probabilities": { "account": 0.031508, "billing": 0.014441, "delivery": 0.030519, "refund": 0.006218, "technical": 0.917314 } }, "churn_risk": { "probabilities": { "0": 0.00607, "1": 0.072537, "2": 0.237151, "3": 0.684242 } }, "needs_human": { "probabilities": { "false": 0.339126, "true": 0.660874 } }, "urgency": { "probabilities": { "0": 0.010711, "1": 0.106778, "2": 0.553366, "3": 0.329145 } } }, "customer_service_000059": { "action": { "probabilities": { "answer_directly": 0.106923, "close_no_action": 0.018225, "escalate_to_human": 0.655228, "execute_refund": 0.112077, "request_information": 0.107547 } }, "category": { "probabilities": { "account": 0.019377, "billing": 0.019904, "delivery": 0.540051, "refund": 0.312434, "technical": 0.108234 } }, "churn_risk": { "probabilities": { "0": 0.005331, "1": 0.100968, "2": 0.498001, "3": 0.395699 } }, "needs_human": { "probabilities": { "false": 0.217724, "true": 0.782276 } }, "urgency": { "probabilities": { "0": 0.00362, "1": 0.046453, "2": 0.300296, "3": 0.649631 } } }, "customer_service_000060": { "action": { "probabilities": { "answer_directly": 0.41094, "close_no_action": 0.012136, "escalate_to_human": 0.411799, "execute_refund": 0.037156, "request_information": 0.127969 } }, "category": { "probabilities": { "account": 0.020792, "billing": 0.943388, "delivery": 0.004072, "refund": 0.012438, "technical": 0.01931 } }, "churn_risk": { "probabilities": { "0": 0.062545, "1": 0.361388, "2": 0.399433, "3": 0.176633 } }, "needs_human": { "probabilities": { "false": 0.455628, "true": 0.544372 } }, "urgency": { "probabilities": { "0": 0.003311, "1": 0.028225, "2": 0.093352, "3": 0.875112 } } }, "customer_service_000061": { "action": { "probabilities": { "answer_directly": 0.296783, "close_no_action": 0.014503, "escalate_to_human": 0.470633, "execute_refund": 0.030243, "request_information": 0.187838 } }, "category": { "probabilities": { "account": 0.059639, "billing": 0.858009, "delivery": 0.002923, "refund": 0.009611, "technical": 0.069818 } }, "churn_risk": { "probabilities": { "0": 0.009335, "1": 0.086442, "2": 0.360968, "3": 0.543255 } }, "needs_human": { "probabilities": { "false": 0.381135, "true": 0.618865 } }, "urgency": { "probabilities": { "0": 0.007547, "1": 0.1498, "2": 0.732117, "3": 0.110536 } } }, "customer_service_000062": { "action": { "probabilities": { "answer_directly": 0.317742, "close_no_action": 0.020827, "escalate_to_human": 0.343515, "execute_refund": 0.07337, "request_information": 0.244546 } }, "category": { "probabilities": { "account": 0.001002, "billing": 0.00268, "delivery": 0.970967, "refund": 0.022546, "technical": 0.002805 } }, "churn_risk": { "probabilities": { "0": 0.148874, "1": 0.566912, "2": 0.260733, "3": 0.023481 } }, "needs_human": { "probabilities": { "false": 0.548218, "true": 0.451782 } }, "urgency": { "probabilities": { "0": 0.022576, "1": 0.530707, "2": 0.389145, "3": 0.057572 } } }, "customer_service_000063": { "action": { "probabilities": { "answer_directly": 0.432113, "close_no_action": 0.013593, "escalate_to_human": 0.217572, "execute_refund": 0.093158, "request_information": 0.243565 } }, "category": { "probabilities": { "account": 0.009873, "billing": 0.936911, "delivery": 0.002237, "refund": 0.044845, "technical": 0.006134 } }, "churn_risk": { "probabilities": { "0": 0.255749, "1": 0.567322, "2": 0.164092, "3": 0.012837 } }, "needs_human": { "probabilities": { "false": 0.654923, "true": 0.345077 } }, "urgency": { "probabilities": { "0": 0.075452, "1": 0.711095, "2": 0.178719, "3": 0.034734 } } }, "customer_service_000064": { "action": { "probabilities": { "answer_directly": 0.119799, "close_no_action": 0.015742, "escalate_to_human": 0.640948, "execute_refund": 0.009001, "request_information": 0.21451 } }, "category": { "probabilities": { "account": 0.060309, "billing": 0.016153, "delivery": 0.026938, "refund": 0.00472, "technical": 0.89188 } }, "churn_risk": { "probabilities": { "0": 0.061619, "1": 0.443761, "2": 0.407631, "3": 0.086989 } }, "needs_human": { "probabilities": { "false": 0.241981, "true": 0.758019 } }, "urgency": { "probabilities": { "0": 0.005788, "1": 0.090893, "2": 0.588842, "3": 0.314477 } } }, "customer_service_000065": { "action": { "probabilities": { "answer_directly": 0.134128, "close_no_action": 0.012051, "escalate_to_human": 0.55251, "execute_refund": 0.00609, "request_information": 0.295221 } }, "category": { "probabilities": { "account": 0.307042, "billing": 0.02856, "delivery": 0.005753, "refund": 0.004196, "technical": 0.654449 } }, "churn_risk": { "probabilities": { "0": 0.003677, "1": 0.048967, "2": 0.164313, "3": 0.783043 } }, "needs_human": { "probabilities": { "false": 0.240013, "true": 0.759987 } }, "urgency": { "probabilities": { "0": 0.005895, "1": 0.07197, "2": 0.437457, "3": 0.484678 } } }, "customer_service_000066": { "action": { "probabilities": { "answer_directly": 0.226214, "close_no_action": 0.009105, "escalate_to_human": 0.465428, "execute_refund": 0.015606, "request_information": 0.283648 } }, "category": { "probabilities": { "account": 0.150209, "billing": 0.752604, "delivery": 0.002304, "refund": 0.007721, "technical": 0.087162 } }, "churn_risk": { "probabilities": { "0": 0.021669, "1": 0.262393, "2": 0.536836, "3": 0.179102 } }, "needs_human": { "probabilities": { "false": 0.378703, "true": 0.621297 } }, "urgency": { "probabilities": { "0": 0.007966, "1": 0.117525, "2": 0.353986, "3": 0.520523 } } }, "customer_service_000067": { "action": { "probabilities": { "answer_directly": 0.158907, "close_no_action": 0.014457, "escalate_to_human": 0.479008, "execute_refund": 0.023783, "request_information": 0.323846 } }, "category": { "probabilities": { "account": 0.007901, "billing": 0.016729, "delivery": 0.915243, "refund": 0.035544, "technical": 0.024583 } }, "churn_risk": { "probabilities": { "0": 0.048297, "1": 0.412286, "2": 0.491952, "3": 0.047465 } }, "needs_human": { "probabilities": { "false": 0.347693, "true": 0.652307 } }, "urgency": { "probabilities": { "0": 0.029111, "1": 0.384446, "2": 0.48164, "3": 0.104803 } } }, "customer_service_000068": { "action": { "probabilities": { "answer_directly": 0.530243, "close_no_action": 0.025425, "escalate_to_human": 0.273636, "execute_refund": 0.020653, "request_information": 0.150043 } }, "category": { "probabilities": { "account": 0.004097, "billing": 0.008009, "delivery": 0.978551, "refund": 0.006549, "technical": 0.002794 } }, "churn_risk": { "probabilities": { "0": 0.107618, "1": 0.375956, "2": 0.468636, "3": 0.04779 } }, "needs_human": { "probabilities": { "false": 0.642044, "true": 0.357956 } }, "urgency": { "probabilities": { "0": 0.021097, "1": 0.399247, "2": 0.530795, "3": 0.048861 } } }, "customer_service_000069": { "action": { "probabilities": { "answer_directly": 0.191066, "close_no_action": 0.016852, "escalate_to_human": 0.536725, "execute_refund": 0.103892, "request_information": 0.151465 } }, "category": { "probabilities": { "account": 0.021515, "billing": 0.909993, "delivery": 0.00568, "refund": 0.041763, "technical": 0.021049 } }, "churn_risk": { "probabilities": { "0": 0.016832, "1": 0.170948, "2": 0.60696, "3": 0.20526 } }, "needs_human": { "probabilities": { "false": 0.320939, "true": 0.679061 } }, "urgency": { "probabilities": { "0": 0.013092, "1": 0.136004, "2": 0.431015, "3": 0.419888 } } }, "customer_service_000070": { "action": { "probabilities": { "answer_directly": 0.251025, "close_no_action": 0.009089, "escalate_to_human": 0.342078, "execute_refund": 0.00182, "request_information": 0.395988 } }, "category": { "probabilities": { "account": 0.045955, "billing": 0.011053, "delivery": 0.008091, "refund": 0.001613, "technical": 0.933288 } }, "churn_risk": { "probabilities": { "0": 0.12853, "1": 0.534728, "2": 0.286068, "3": 0.050674 } }, "needs_human": { "probabilities": { "false": 0.517449, "true": 0.482551 } }, "urgency": { "probabilities": { "0": 0.037837, "1": 0.476226, "2": 0.389673, "3": 0.096264 } } }, "customer_service_000071": { "action": { "probabilities": { "answer_directly": 0.297494, "close_no_action": 0.009989, "escalate_to_human": 0.41067, "execute_refund": 0.001816, "request_information": 0.280032 } }, "category": { "probabilities": { "account": 0.841042, "billing": 0.029684, "delivery": 0.00187, "refund": 0.001691, "technical": 0.125713 } }, "churn_risk": { "probabilities": { "0": 0.060287, "1": 0.564276, "2": 0.343256, "3": 0.032181 } }, "needs_human": { "probabilities": { "false": 0.447178, "true": 0.552822 } }, "urgency": { "probabilities": { "0": 0.024928, "1": 0.310839, "2": 0.413877, "3": 0.250357 } } }, "customer_service_000072": { "action": { "probabilities": { "answer_directly": 0.253296, "close_no_action": 0.012004, "escalate_to_human": 0.624934, "execute_refund": 0.030553, "request_information": 0.079213 } }, "category": { "probabilities": { "account": 0.086477, "billing": 0.847542, "delivery": 0.008616, "refund": 0.01289, "technical": 0.044475 } }, "churn_risk": { "probabilities": { "0": 0.012092, "1": 0.100559, "2": 0.38686, "3": 0.500489 } }, "needs_human": { "probabilities": { "false": 0.265806, "true": 0.734194 } }, "urgency": { "probabilities": { "0": 0.003479, "1": 0.024861, "2": 0.1062, "3": 0.86546 } } }, "customer_service_000073": { "action": { "probabilities": { "answer_directly": 0.166828, "close_no_action": 0.011676, "escalate_to_human": 0.470707, "execute_refund": 0.005451, "request_information": 0.345338 } }, "category": { "probabilities": { "account": 0.038761, "billing": 0.012121, "delivery": 0.009717, "refund": 0.002708, "technical": 0.936693 } }, "churn_risk": { "probabilities": { "0": 0.019136, "1": 0.246697, "2": 0.427498, "3": 0.306669 } }, "needs_human": { "probabilities": { "false": 0.336771, "true": 0.663229 } }, "urgency": { "probabilities": { "0": 0.005256, "1": 0.061684, "2": 0.323965, "3": 0.609095 } } }, "customer_service_000074": { "action": { "probabilities": { "answer_directly": 0.306086, "close_no_action": 0.028228, "escalate_to_human": 0.461767, "execute_refund": 0.021893, "request_information": 0.182026 } }, "category": { "probabilities": { "account": 0.013691, "billing": 0.010606, "delivery": 0.941684, "refund": 0.011632, "technical": 0.022387 } }, "churn_risk": { "probabilities": { "0": 0.096671, "1": 0.391085, "2": 0.404291, "3": 0.107953 } }, "needs_human": { "probabilities": { "false": 0.38939, "true": 0.61061 } }, "urgency": { "probabilities": { "0": 0.006169, "1": 0.08184, "2": 0.595461, "3": 0.31653 } } }, "customer_service_000075": { "action": { "probabilities": { "answer_directly": 0.112038, "close_no_action": 0.024592, "escalate_to_human": 0.618624, "execute_refund": 0.029013, "request_information": 0.215733 } }, "category": { "probabilities": { "account": 0.021156, "billing": 0.007322, "delivery": 0.435036, "refund": 0.024291, "technical": 0.512195 } }, "churn_risk": { "probabilities": { "0": 0.006543, "1": 0.064359, "2": 0.259782, "3": 0.669316 } }, "needs_human": { "probabilities": { "false": 0.246782, "true": 0.753218 } }, "urgency": { "probabilities": { "0": 0.004907, "1": 0.040423, "2": 0.113128, "3": 0.841542 } } }, "customer_service_000076": { "action": { "probabilities": { "answer_directly": 0.218621, "close_no_action": 0.023358, "escalate_to_human": 0.604157, "execute_refund": 0.044744, "request_information": 0.10912 } }, "category": { "probabilities": { "account": 0.013919, "billing": 0.012547, "delivery": 0.952154, "refund": 0.015337, "technical": 0.006043 } }, "churn_risk": { "probabilities": { "0": 0.008986, "1": 0.091175, "2": 0.48931, "3": 0.410529 } }, "needs_human": { "probabilities": { "false": 0.281063, "true": 0.718937 } }, "urgency": { "probabilities": { "0": 0.007379, "1": 0.101612, "2": 0.498393, "3": 0.392616 } } }, "customer_service_000077": { "action": { "probabilities": { "answer_directly": 0.101964, "close_no_action": 0.010867, "escalate_to_human": 0.737167, "execute_refund": 0.020348, "request_information": 0.129654 } }, "category": { "probabilities": { "account": 0.622743, "billing": 0.301674, "delivery": 0.002143, "refund": 0.007104, "technical": 0.066336 } }, "churn_risk": { "probabilities": { "0": 0.002225, "1": 0.061229, "2": 0.344001, "3": 0.592545 } }, "needs_human": { "probabilities": { "false": 0.158793, "true": 0.841207 } }, "urgency": { "probabilities": { "0": 0.003071, "1": 0.034042, "2": 0.198336, "3": 0.764551 } } }, "customer_service_000078": { "action": { "probabilities": { "answer_directly": 0.068974, "close_no_action": 0.016961, "escalate_to_human": 0.774936, "execute_refund": 0.024306, "request_information": 0.114823 } }, "category": { "probabilities": { "account": 0.062377, "billing": 0.022329, "delivery": 0.033676, "refund": 0.01403, "technical": 0.867588 } }, "churn_risk": { "probabilities": { "0": 0.007726, "1": 0.118598, "2": 0.651142, "3": 0.222534 } }, "needs_human": { "probabilities": { "false": 0.178247, "true": 0.821753 } }, "urgency": { "probabilities": { "0": 0.007154, "1": 0.053432, "2": 0.224791, "3": 0.714623 } } }, "customer_service_000079": { "action": { "probabilities": { "answer_directly": 0.148606, "close_no_action": 0.020151, "escalate_to_human": 0.662455, "execute_refund": 0.053897, "request_information": 0.114891 } }, "category": { "probabilities": { "account": 0.051315, "billing": 0.845012, "delivery": 0.010116, "refund": 0.023486, "technical": 0.070071 } }, "churn_risk": { "probabilities": { "0": 0.023476, "1": 0.177647, "2": 0.513319, "3": 0.285558 } }, "needs_human": { "probabilities": { "false": 0.255038, "true": 0.744962 } }, "urgency": { "probabilities": { "0": 0.012801, "1": 0.129018, "2": 0.424086, "3": 0.434096 } } }, "customer_service_000080": { "action": { "probabilities": { "answer_directly": 0.216682, "close_no_action": 0.015439, "escalate_to_human": 0.437486, "execute_refund": 0.01985, "request_information": 0.310542 } }, "category": { "probabilities": { "account": 0.009008, "billing": 0.010568, "delivery": 0.867915, "refund": 0.025416, "technical": 0.087093 } }, "churn_risk": { "probabilities": { "0": 0.053897, "1": 0.342012, "2": 0.463361, "3": 0.14073 } }, "needs_human": { "probabilities": { "false": 0.372482, "true": 0.627518 } }, "urgency": { "probabilities": { "0": 0.005614, "1": 0.039712, "2": 0.107812, "3": 0.846862 } } }, "customer_service_000081": { "action": { "probabilities": { "answer_directly": 0.239559, "close_no_action": 0.012945, "escalate_to_human": 0.232786, "execute_refund": 0.001517, "request_information": 0.513193 } }, "category": { "probabilities": { "account": 0.23224, "billing": 0.018327, "delivery": 0.004076, "refund": 0.002611, "technical": 0.742746 } }, "churn_risk": { "probabilities": { "0": 0.18059, "1": 0.577893, "2": 0.231822, "3": 0.009695 } }, "needs_human": { "probabilities": { "false": 0.631277, "true": 0.368723 } }, "urgency": { "probabilities": { "0": 0.104451, "1": 0.631821, "2": 0.216545, "3": 0.047183 } } }, "customer_service_000082": { "action": { "probabilities": { "answer_directly": 0.167813, "close_no_action": 0.017528, "escalate_to_human": 0.68021, "execute_refund": 0.018088, "request_information": 0.116361 } }, "category": { "probabilities": { "account": 0.341598, "billing": 0.624623, "delivery": 0.000989, "refund": 0.00478, "technical": 0.02801 } }, "churn_risk": { "probabilities": { "0": 0.024363, "1": 0.170974, "2": 0.62646, "3": 0.178203 } }, "needs_human": { "probabilities": { "false": 0.260531, "true": 0.739469 } }, "urgency": { "probabilities": { "0": 0.031582, "1": 0.264461, "2": 0.553745, "3": 0.150212 } } }, "customer_service_000083": { "action": { "probabilities": { "answer_directly": 0.238437, "close_no_action": 0.007634, "escalate_to_human": 0.365255, "execute_refund": 0.001549, "request_information": 0.387125 } }, "category": { "probabilities": { "account": 0.822863, "billing": 0.023238, "delivery": 0.001644, "refund": 0.001522, "technical": 0.150733 } }, "churn_risk": { "probabilities": { "0": 0.019998, "1": 0.270815, "2": 0.519811, "3": 0.189376 } }, "needs_human": { "probabilities": { "false": 0.443362, "true": 0.556638 } }, "urgency": { "probabilities": { "0": 0.025948, "1": 0.314943, "2": 0.447666, "3": 0.211443 } } }, "customer_service_000084": { "action": { "probabilities": { "answer_directly": 0.090678, "close_no_action": 0.008082, "escalate_to_human": 0.612835, "execute_refund": 0.200581, "request_information": 0.087824 } }, "category": { "probabilities": { "account": 0.023362, "billing": 0.054295, "delivery": 0.015226, "refund": 0.902169, "technical": 0.004948 } }, "churn_risk": { "probabilities": { "0": 0.007021, "1": 0.10317, "2": 0.512908, "3": 0.376901 } }, "needs_human": { "probabilities": { "false": 0.176291, "true": 0.823709 } }, "urgency": { "probabilities": { "0": 0.006149, "1": 0.085931, "2": 0.411302, "3": 0.496617 } } }, "customer_service_000085": { "action": { "probabilities": { "answer_directly": 0.1096, "close_no_action": 0.022889, "escalate_to_human": 0.675219, "execute_refund": 0.046086, "request_information": 0.146206 } }, "category": { "probabilities": { "account": 0.015549, "billing": 0.007042, "delivery": 0.934231, "refund": 0.021556, "technical": 0.021622 } }, "churn_risk": { "probabilities": { "0": 0.0047, "1": 0.053137, "2": 0.218333, "3": 0.72383 } }, "needs_human": { "probabilities": { "false": 0.172469, "true": 0.827531 } }, "urgency": { "probabilities": { "0": 0.00506, "1": 0.071694, "2": 0.323907, "3": 0.599339 } } }, "customer_service_000086": { "action": { "probabilities": { "answer_directly": 0.051724, "close_no_action": 0.011143, "escalate_to_human": 0.556718, "execute_refund": 0.304257, "request_information": 0.076158 } }, "category": { "probabilities": { "account": 0.026504, "billing": 0.747451, "delivery": 0.010657, "refund": 0.199383, "technical": 0.016005 } }, "churn_risk": { "probabilities": { "0": 0.009921, "1": 0.124935, "2": 0.60063, "3": 0.264514 } }, "needs_human": { "probabilities": { "false": 0.154956, "true": 0.845044 } }, "urgency": { "probabilities": { "0": 0.011754, "1": 0.150864, "2": 0.618166, "3": 0.219216 } } }, "customer_service_000087": { "action": { "probabilities": { "answer_directly": 0.060728, "close_no_action": 0.011725, "escalate_to_human": 0.791877, "execute_refund": 0.002465, "request_information": 0.133205 } }, "category": { "probabilities": { "account": 0.631828, "billing": 0.018609, "delivery": 0.002312, "refund": 0.001539, "technical": 0.345712 } }, "churn_risk": { "probabilities": { "0": 0.009237, "1": 0.138501, "2": 0.615516, "3": 0.236746 } }, "needs_human": { "probabilities": { "false": 0.147782, "true": 0.852218 } }, "urgency": { "probabilities": { "0": 0.01583, "1": 0.257091, "2": 0.649831, "3": 0.077248 } } }, "customer_service_000088": { "action": { "probabilities": { "answer_directly": 0.222814, "close_no_action": 0.017602, "escalate_to_human": 0.581611, "execute_refund": 0.04034, "request_information": 0.137633 } }, "category": { "probabilities": { "account": 0.084375, "billing": 0.8616, "delivery": 0.005558, "refund": 0.013461, "technical": 0.035006 } }, "churn_risk": { "probabilities": { "0": 0.027645, "1": 0.179687, "2": 0.562226, "3": 0.230442 } }, "needs_human": { "probabilities": { "false": 0.342214, "true": 0.657786 } }, "urgency": { "probabilities": { "0": 0.014781, "1": 0.168874, "2": 0.582786, "3": 0.233559 } } }, "customer_service_000089": { "action": { "probabilities": { "answer_directly": 0.919616, "close_no_action": 0.018383, "escalate_to_human": 0.03636, "execute_refund": 0.001969, "request_information": 0.023672 } }, "category": { "probabilities": { "account": 0.945456, "billing": 0.033538, "delivery": 0.001096, "refund": 0.004511, "technical": 0.015399 } }, "churn_risk": { "probabilities": { "0": 0.067796, "1": 0.037594, "2": 0.027551, "3": 0.867059 } }, "needs_human": { "probabilities": { "false": 0.883814, "true": 0.116186 } }, "urgency": { "probabilities": { "0": 0.141353, "1": 0.571819, "2": 0.204945, "3": 0.081883 } } }, "customer_service_000090": { "action": { "probabilities": { "answer_directly": 0.09829, "close_no_action": 0.019511, "escalate_to_human": 0.693271, "execute_refund": 0.044701, "request_information": 0.144227 } }, "category": { "probabilities": { "account": 0.317405, "billing": 0.06766, "delivery": 0.020842, "refund": 0.176211, "technical": 0.417881 } }, "churn_risk": { "probabilities": { "0": 0.001229, "1": 0.016355, "2": 0.114516, "3": 0.8679 } }, "needs_human": { "probabilities": { "false": 0.160706, "true": 0.839294 } }, "urgency": { "probabilities": { "0": 0.005167, "1": 0.040699, "2": 0.228276, "3": 0.725858 } } }, "customer_service_000091": { "action": { "probabilities": { "answer_directly": 0.14847, "close_no_action": 0.01903, "escalate_to_human": 0.638372, "execute_refund": 0.070183, "request_information": 0.123945 } }, "category": { "probabilities": { "account": 0.041289, "billing": 0.916947, "delivery": 0.005154, "refund": 0.021338, "technical": 0.015272 } }, "churn_risk": { "probabilities": { "0": 0.030817, "1": 0.194748, "2": 0.522965, "3": 0.25147 } }, "needs_human": { "probabilities": { "false": 0.248328, "true": 0.751672 } }, "urgency": { "probabilities": { "0": 0.01813, "1": 0.153106, "2": 0.398299, "3": 0.430464 } } }, "customer_service_000092": { "action": { "probabilities": { "answer_directly": 0.089496, "close_no_action": 0.015108, "escalate_to_human": 0.525952, "execute_refund": 0.043642, "request_information": 0.325802 } }, "category": { "probabilities": { "account": 0.003336, "billing": 0.008024, "delivery": 0.946282, "refund": 0.032417, "technical": 0.009941 } }, "churn_risk": { "probabilities": { "0": 0.028972, "1": 0.260003, "2": 0.636008, "3": 0.075017 } }, "needs_human": { "probabilities": { "false": 0.257758, "true": 0.742242 } }, "urgency": { "probabilities": { "0": 0.019568, "1": 0.238086, "2": 0.584141, "3": 0.158205 } } }, "customer_service_000093": { "action": { "probabilities": { "answer_directly": 0.064712, "close_no_action": 0.013759, "escalate_to_human": 0.77806, "execute_refund": 0.014865, "request_information": 0.128604 } }, "category": { "probabilities": { "account": 0.099143, "billing": 0.020217, "delivery": 0.013837, "refund": 0.007436, "technical": 0.859367 } }, "churn_risk": { "probabilities": { "0": 0.002373, "1": 0.038717, "2": 0.253387, "3": 0.705523 } }, "needs_human": { "probabilities": { "false": 0.12445, "true": 0.87555 } }, "urgency": { "probabilities": { "0": 0.002772, "1": 0.020863, "2": 0.114655, "3": 0.86171 } } }, "customer_service_000094": { "action": { "probabilities": { "answer_directly": 0.167264, "close_no_action": 0.011787, "escalate_to_human": 0.555225, "execute_refund": 0.081069, "request_information": 0.184655 } }, "category": { "probabilities": { "account": 0.026341, "billing": 0.868467, "delivery": 0.016435, "refund": 0.061352, "technical": 0.027405 } }, "churn_risk": { "probabilities": { "0": 0.022705, "1": 0.199766, "2": 0.633913, "3": 0.143616 } }, "needs_human": { "probabilities": { "false": 0.287122, "true": 0.712878 } }, "urgency": { "probabilities": { "0": 0.006231, "1": 0.041248, "2": 0.127904, "3": 0.824617 } } }, "customer_service_000095": { "action": { "probabilities": { "answer_directly": 0.254904, "close_no_action": 0.015568, "escalate_to_human": 0.34547, "execute_refund": 0.046071, "request_information": 0.337988 } }, "category": { "probabilities": { "account": 0.003027, "billing": 0.004171, "delivery": 0.934261, "refund": 0.046004, "technical": 0.012537 } }, "churn_risk": { "probabilities": { "0": 0.070056, "1": 0.516536, "2": 0.36342, "3": 0.049988 } }, "needs_human": { "probabilities": { "false": 0.492886, "true": 0.507114 } }, "urgency": { "probabilities": { "0": 0.032616, "1": 0.506401, "2": 0.382004, "3": 0.078979 } } }, "customer_service_000096": { "action": { "probabilities": { "answer_directly": 0.171957, "close_no_action": 0.014488, "escalate_to_human": 0.415523, "execute_refund": 0.272761, "request_information": 0.125271 } }, "category": { "probabilities": { "account": 0.038962, "billing": 0.104111, "delivery": 0.013872, "refund": 0.835735, "technical": 0.00732 } }, "churn_risk": { "probabilities": { "0": 0.002312, "1": 0.038391, "2": 0.185489, "3": 0.773808 } }, "needs_human": { "probabilities": { "false": 0.313325, "true": 0.686675 } }, "urgency": { "probabilities": { "0": 0.003412, "1": 0.044246, "2": 0.292643, "3": 0.659699 } } }, "customer_service_000097": { "action": { "probabilities": { "answer_directly": 0.068475, "close_no_action": 0.008868, "escalate_to_human": 0.772057, "execute_refund": 0.008881, "request_information": 0.141719 } }, "category": { "probabilities": { "account": 0.064714, "billing": 0.013025, "delivery": 0.017811, "refund": 0.005486, "technical": 0.898964 } }, "churn_risk": { "probabilities": { "0": 0.003889, "1": 0.074584, "2": 0.300126, "3": 0.621401 } }, "needs_human": { "probabilities": { "false": 0.123782, "true": 0.876218 } }, "urgency": { "probabilities": { "0": 0.003382, "1": 0.051862, "2": 0.480853, "3": 0.463903 } } }, "customer_service_000098": { "action": { "probabilities": { "answer_directly": 0.222849, "close_no_action": 0.014762, "escalate_to_human": 0.524944, "execute_refund": 0.115286, "request_information": 0.122159 } }, "category": { "probabilities": { "account": 0.01721, "billing": 0.934225, "delivery": 0.00657, "refund": 0.025998, "technical": 0.015997 } }, "churn_risk": { "probabilities": { "0": 0.015349, "1": 0.109083, "2": 0.476778, "3": 0.39879 } }, "needs_human": { "probabilities": { "false": 0.304226, "true": 0.695774 } }, "urgency": { "probabilities": { "0": 0.003983, "1": 0.036502, "2": 0.137814, "3": 0.821701 } } }, "customer_service_000099": { "action": { "probabilities": { "answer_directly": 0.692585, "close_no_action": 0.012849, "escalate_to_human": 0.227998, "execute_refund": 0.00499, "request_information": 0.061578 } }, "category": { "probabilities": { "account": 0.78397, "billing": 0.178358, "delivery": 0.002954, "refund": 0.00198, "technical": 0.032738 } }, "churn_risk": { "probabilities": { "0": 0.121609, "1": 0.486135, "2": 0.357038, "3": 0.035218 } }, "needs_human": { "probabilities": { "false": 0.655798, "true": 0.344202 } }, "urgency": { "probabilities": { "0": 0.005985, "1": 0.043477, "2": 0.131114, "3": 0.819424 } } }, "invoice_processing_000000": { "discrepancy_severity": { "probabilities": { "0": 0.497498, "1": 0.053338, "2": 0.155808, "3": 0.293356 } }, "disposition": { "probabilities": { "approve": 0.464632, "hold": 0.1166, "manual_review": 0.362447, "reject": 0.05632 } }, "duplicate": { "probabilities": { "false": 0.965104, "true": 0.034896 } }, "matches_order": { "probabilities": { "false": 0.397487, "true": 0.602513 } }, "urgency": { "probabilities": { "0": 0.022887, "1": 0.188682, "2": 0.508687, "3": 0.279744 } } }, "invoice_processing_000001": { "discrepancy_severity": { "probabilities": { "0": 0.022343, "1": 0.019126, "2": 0.136245, "3": 0.822286 } }, "disposition": { "probabilities": { "approve": 0.046995, "hold": 0.188085, "manual_review": 0.690841, "reject": 0.074079 } }, "duplicate": { "probabilities": { "false": 0.971788, "true": 0.028212 } }, "matches_order": { "probabilities": { "false": 0.953291, "true": 0.046709 } }, "urgency": { "probabilities": { "0": 0.046876, "1": 0.608065, "2": 0.294225, "3": 0.050834 } } }, "invoice_processing_000002": { "discrepancy_severity": { "probabilities": { "0": 0.895976, "1": 0.032777, "2": 0.051463, "3": 0.019784 } }, "disposition": { "probabilities": { "approve": 0.859048, "hold": 0.072482, "manual_review": 0.046481, "reject": 0.021989 } }, "duplicate": { "probabilities": { "false": 0.959519, "true": 0.040481 } }, "matches_order": { "probabilities": { "false": 0.074825, "true": 0.925175 } }, "urgency": { "probabilities": { "0": 0.038975, "1": 0.401934, "2": 0.377975, "3": 0.181116 } } }, "invoice_processing_000003": { "discrepancy_severity": { "probabilities": { "0": 0.008244, "1": 0.074935, "2": 0.72128, "3": 0.195541 } }, "disposition": { "probabilities": { "approve": 0.012652, "hold": 0.293623, "manual_review": 0.668111, "reject": 0.025614 } }, "duplicate": { "probabilities": { "false": 0.970572, "true": 0.029428 } }, "matches_order": { "probabilities": { "false": 0.978921, "true": 0.021079 } }, "urgency": { "probabilities": { "0": 0.020571, "1": 0.147841, "2": 0.499944, "3": 0.331645 } } }, "invoice_processing_000004": { "discrepancy_severity": { "probabilities": { "0": 0.031288, "1": 0.01507, "2": 0.091559, "3": 0.862083 } }, "disposition": { "probabilities": { "approve": 0.05264, "hold": 0.194734, "manual_review": 0.664896, "reject": 0.08773 } }, "duplicate": { "probabilities": { "false": 0.983576, "true": 0.016424 } }, "matches_order": { "probabilities": { "false": 0.933385, "true": 0.066615 } }, "urgency": { "probabilities": { "0": 0.056752, "1": 0.511597, "2": 0.374381, "3": 0.05727 } } }, "invoice_processing_000005": { "discrepancy_severity": { "probabilities": { "0": 0.375367, "1": 0.046768, "2": 0.182678, "3": 0.395186 } }, "disposition": { "probabilities": { "approve": 0.352125, "hold": 0.165296, "manual_review": 0.413026, "reject": 0.069552 } }, "duplicate": { "probabilities": { "false": 0.973981, "true": 0.026019 } }, "matches_order": { "probabilities": { "false": 0.536875, "true": 0.463125 } }, "urgency": { "probabilities": { "0": 0.024567, "1": 0.219772, "2": 0.555572, "3": 0.200089 } } }, "invoice_processing_000006": { "discrepancy_severity": { "probabilities": { "0": 0.058786, "1": 0.117344, "2": 0.720831, "3": 0.103039 } }, "disposition": { "probabilities": { "approve": 0.137267, "hold": 0.389091, "manual_review": 0.456853, "reject": 0.016788 } }, "duplicate": { "probabilities": { "false": 0.982937, "true": 0.017063 } }, "matches_order": { "probabilities": { "false": 0.845886, "true": 0.154114 } }, "urgency": { "probabilities": { "0": 0.041131, "1": 0.593239, "2": 0.318349, "3": 0.047281 } } }, "invoice_processing_000007": { "discrepancy_severity": { "probabilities": { "0": 0.072567, "1": 0.048671, "2": 0.277595, "3": 0.601167 } }, "disposition": { "probabilities": { "approve": 0.08616, "hold": 0.201458, "manual_review": 0.662787, "reject": 0.049595 } }, "duplicate": { "probabilities": { "false": 0.974558, "true": 0.025442 } }, "matches_order": { "probabilities": { "false": 0.897085, "true": 0.102915 } }, "urgency": { "probabilities": { "0": 0.0219, "1": 0.14606, "2": 0.490159, "3": 0.34188 } } }, "invoice_processing_000008": { "discrepancy_severity": { "probabilities": { "0": 0.016007, "1": 0.038801, "2": 0.295328, "3": 0.649864 } }, "disposition": { "probabilities": { "approve": 0.021434, "hold": 0.570201, "manual_review": 0.372465, "reject": 0.0359 } }, "duplicate": { "probabilities": { "false": 0.961742, "true": 0.038258 } }, "matches_order": { "probabilities": { "false": 0.947174, "true": 0.052826 } }, "urgency": { "probabilities": { "0": 0.068919, "1": 0.373839, "2": 0.477902, "3": 0.079339 } } }, "invoice_processing_000009": { "discrepancy_severity": { "probabilities": { "0": 0.426396, "1": 0.167378, "2": 0.341455, "3": 0.064772 } }, "disposition": { "probabilities": { "approve": 0.416193, "hold": 0.367497, "manual_review": 0.193687, "reject": 0.022624 } }, "duplicate": { "probabilities": { "false": 0.972106, "true": 0.027894 } }, "matches_order": { "probabilities": { "false": 0.311525, "true": 0.688475 } }, "urgency": { "probabilities": { "0": 0.056735, "1": 0.482922, "2": 0.372673, "3": 0.08767 } } }, "invoice_processing_000010": { "discrepancy_severity": { "probabilities": { "0": 0.887452, "1": 0.037602, "2": 0.054876, "3": 0.02007 } }, "disposition": { "probabilities": { "approve": 0.842612, "hold": 0.088599, "manual_review": 0.048325, "reject": 0.020464 } }, "duplicate": { "probabilities": { "false": 0.958739, "true": 0.041261 } }, "matches_order": { "probabilities": { "false": 0.074304, "true": 0.925696 } }, "urgency": { "probabilities": { "0": 0.036425, "1": 0.340482, "2": 0.446914, "3": 0.176179 } } }, "invoice_processing_000011": { "discrepancy_severity": { "probabilities": { "0": 0.868631, "1": 0.039055, "2": 0.070428, "3": 0.021886 } }, "disposition": { "probabilities": { "approve": 0.835652, "hold": 0.090332, "manual_review": 0.054512, "reject": 0.019504 } }, "duplicate": { "probabilities": { "false": 0.967266, "true": 0.032734 } }, "matches_order": { "probabilities": { "false": 0.106224, "true": 0.893776 } }, "urgency": { "probabilities": { "0": 0.035833, "1": 0.38462, "2": 0.366648, "3": 0.212899 } } }, "invoice_processing_000012": { "discrepancy_severity": { "probabilities": { "0": 0.815114, "1": 0.057842, "2": 0.079704, "3": 0.04734 } }, "disposition": { "probabilities": { "approve": 0.641269, "hold": 0.123424, "manual_review": 0.132108, "reject": 0.103199 } }, "duplicate": { "probabilities": { "false": 0.695382, "true": 0.304618 } }, "matches_order": { "probabilities": { "false": 0.092726, "true": 0.907274 } }, "urgency": { "probabilities": { "0": 0.011101, "1": 0.087428, "2": 0.465684, "3": 0.435787 } } }, "invoice_processing_000013": { "discrepancy_severity": { "probabilities": { "0": 0.028519, "1": 0.043312, "2": 0.30465, "3": 0.623519 } }, "disposition": { "probabilities": { "approve": 0.044208, "hold": 0.210492, "manual_review": 0.702616, "reject": 0.042684 } }, "duplicate": { "probabilities": { "false": 0.966872, "true": 0.033128 } }, "matches_order": { "probabilities": { "false": 0.95778, "true": 0.04222 } }, "urgency": { "probabilities": { "0": 0.047901, "1": 0.429633, "2": 0.437049, "3": 0.085416 } } }, "invoice_processing_000014": { "discrepancy_severity": { "probabilities": { "0": 0.776153, "1": 0.058077, "2": 0.109919, "3": 0.055851 } }, "disposition": { "probabilities": { "approve": 0.689698, "hold": 0.129267, "manual_review": 0.135337, "reject": 0.045698 } }, "duplicate": { "probabilities": { "false": 0.916032, "true": 0.083968 } }, "matches_order": { "probabilities": { "false": 0.185855, "true": 0.814145 } }, "urgency": { "probabilities": { "0": 0.041722, "1": 0.371462, "2": 0.403995, "3": 0.182822 } } }, "invoice_processing_000015": { "discrepancy_severity": { "probabilities": { "0": 0.828564, "1": 0.050757, "2": 0.079192, "3": 0.041487 } }, "disposition": { "probabilities": { "approve": 0.732103, "hold": 0.11945, "manual_review": 0.106224, "reject": 0.042223 } }, "duplicate": { "probabilities": { "false": 0.903795, "true": 0.096205 } }, "matches_order": { "probabilities": { "false": 0.115245, "true": 0.884755 } }, "urgency": { "probabilities": { "0": 0.049601, "1": 0.374954, "2": 0.452954, "3": 0.122491 } } }, "invoice_processing_000016": { "discrepancy_severity": { "probabilities": { "0": 0.021012, "1": 0.012233, "2": 0.092476, "3": 0.874279 } }, "disposition": { "probabilities": { "approve": 0.036837, "hold": 0.187318, "manual_review": 0.710707, "reject": 0.065138 } }, "duplicate": { "probabilities": { "false": 0.979738, "true": 0.020262 } }, "matches_order": { "probabilities": { "false": 0.961324, "true": 0.038676 } }, "urgency": { "probabilities": { "0": 0.047218, "1": 0.305448, "2": 0.426563, "3": 0.220771 } } }, "invoice_processing_000017": { "discrepancy_severity": { "probabilities": { "0": 0.018842, "1": 0.021235, "2": 0.188892, "3": 0.771031 } }, "disposition": { "probabilities": { "approve": 0.038072, "hold": 0.209912, "manual_review": 0.693177, "reject": 0.058839 } }, "duplicate": { "probabilities": { "false": 0.969348, "true": 0.030652 } }, "matches_order": { "probabilities": { "false": 0.964583, "true": 0.035417 } }, "urgency": { "probabilities": { "0": 0.056101, "1": 0.631497, "2": 0.276838, "3": 0.035564 } } }, "invoice_processing_000018": { "discrepancy_severity": { "probabilities": { "0": 0.023084, "1": 0.04682, "2": 0.396192, "3": 0.533904 } }, "disposition": { "probabilities": { "approve": 0.023661, "hold": 0.53416, "manual_review": 0.409365, "reject": 0.032814 } }, "duplicate": { "probabilities": { "false": 0.95967, "true": 0.04033 } }, "matches_order": { "probabilities": { "false": 0.925967, "true": 0.074033 } }, "urgency": { "probabilities": { "0": 0.029437, "1": 0.14116, "2": 0.490114, "3": 0.339289 } } }, "invoice_processing_000019": { "discrepancy_severity": { "probabilities": { "0": 0.006173, "1": 0.052183, "2": 0.546062, "3": 0.395582 } }, "disposition": { "probabilities": { "approve": 0.010011, "hold": 0.329121, "manual_review": 0.632612, "reject": 0.028256 } }, "duplicate": { "probabilities": { "false": 0.975442, "true": 0.024558 } }, "matches_order": { "probabilities": { "false": 0.987161, "true": 0.012839 } }, "urgency": { "probabilities": { "0": 0.023675, "1": 0.17534, "2": 0.564404, "3": 0.236581 } } }, "invoice_processing_000020": { "discrepancy_severity": { "probabilities": { "0": 0.885753, "1": 0.036329, "2": 0.053866, "3": 0.024052 } }, "disposition": { "probabilities": { "approve": 0.862156, "hold": 0.079562, "manual_review": 0.040537, "reject": 0.017745 } }, "duplicate": { "probabilities": { "false": 0.973423, "true": 0.026577 } }, "matches_order": { "probabilities": { "false": 0.085833, "true": 0.914167 } }, "urgency": { "probabilities": { "0": 0.044267, "1": 0.449469, "2": 0.41027, "3": 0.095993 } } }, "invoice_processing_000021": { "discrepancy_severity": { "probabilities": { "0": 0.703301, "1": 0.059922, "2": 0.067686, "3": 0.169091 } }, "disposition": { "probabilities": { "approve": 0.420194, "hold": 0.151151, "manual_review": 0.168335, "reject": 0.260321 } }, "duplicate": { "probabilities": { "false": 0.463481, "true": 0.536519 } }, "matches_order": { "probabilities": { "false": 0.161659, "true": 0.838341 } }, "urgency": { "probabilities": { "0": 0.111165, "1": 0.65583, "2": 0.201194, "3": 0.031811 } } }, "invoice_processing_000022": { "discrepancy_severity": { "probabilities": { "0": 0.383104, "1": 0.175765, "2": 0.349695, "3": 0.091435 } }, "disposition": { "probabilities": { "approve": 0.373655, "hold": 0.338497, "manual_review": 0.24774, "reject": 0.040108 } }, "duplicate": { "probabilities": { "false": 0.918309, "true": 0.081691 } }, "matches_order": { "probabilities": { "false": 0.325891, "true": 0.674109 } }, "urgency": { "probabilities": { "0": 0.049622, "1": 0.361678, "2": 0.439685, "3": 0.149016 } } }, "invoice_processing_000023": { "discrepancy_severity": { "probabilities": { "0": 0.777942, "1": 0.048699, "2": 0.064174, "3": 0.109185 } }, "disposition": { "probabilities": { "approve": 0.58993, "hold": 0.13997, "manual_review": 0.09876, "reject": 0.17134 } }, "duplicate": { "probabilities": { "false": 0.673306, "true": 0.326694 } }, "matches_order": { "probabilities": { "false": 0.135369, "true": 0.864631 } }, "urgency": { "probabilities": { "0": 0.072769, "1": 0.70285, "2": 0.178064, "3": 0.046317 } } }, "invoice_processing_000024": { "discrepancy_severity": { "probabilities": { "0": 0.860052, "1": 0.041041, "2": 0.066965, "3": 0.031942 } }, "disposition": { "probabilities": { "approve": 0.809387, "hold": 0.083316, "manual_review": 0.075932, "reject": 0.031365 } }, "duplicate": { "probabilities": { "false": 0.934839, "true": 0.065161 } }, "matches_order": { "probabilities": { "false": 0.095498, "true": 0.904502 } }, "urgency": { "probabilities": { "0": 0.039298, "1": 0.438682, "2": 0.409428, "3": 0.112592 } } }, "invoice_processing_000025": { "discrepancy_severity": { "probabilities": { "0": 0.005836, "1": 0.051225, "2": 0.517395, "3": 0.425544 } }, "disposition": { "probabilities": { "approve": 0.00882, "hold": 0.355536, "manual_review": 0.613003, "reject": 0.022641 } }, "duplicate": { "probabilities": { "false": 0.97651, "true": 0.02349 } }, "matches_order": { "probabilities": { "false": 0.985837, "true": 0.014163 } }, "urgency": { "probabilities": { "0": 0.036742, "1": 0.257229, "2": 0.578368, "3": 0.127661 } } }, "invoice_processing_000026": { "discrepancy_severity": { "probabilities": { "0": 0.84578, "1": 0.041601, "2": 0.050292, "3": 0.062327 } }, "disposition": { "probabilities": { "approve": 0.707821, "hold": 0.112017, "manual_review": 0.066052, "reject": 0.11411 } }, "duplicate": { "probabilities": { "false": 0.753512, "true": 0.246488 } }, "matches_order": { "probabilities": { "false": 0.092971, "true": 0.907029 } }, "urgency": { "probabilities": { "0": 0.050818, "1": 0.472199, "2": 0.378403, "3": 0.09858 } } }, "invoice_processing_000027": { "discrepancy_severity": { "probabilities": { "0": 0.320411, "1": 0.181108, "2": 0.32494, "3": 0.173542 } }, "disposition": { "probabilities": { "approve": 0.218271, "hold": 0.386256, "manual_review": 0.275842, "reject": 0.11963 } }, "duplicate": { "probabilities": { "false": 0.707582, "true": 0.292418 } }, "matches_order": { "probabilities": { "false": 0.352545, "true": 0.647455 } }, "urgency": { "probabilities": { "0": 0.068982, "1": 0.487284, "2": 0.357416, "3": 0.086318 } } }, "invoice_processing_000028": { "discrepancy_severity": { "probabilities": { "0": 0.070715, "1": 0.032838, "2": 0.253695, "3": 0.642752 } }, "disposition": { "probabilities": { "approve": 0.074557, "hold": 0.483772, "manual_review": 0.385296, "reject": 0.056376 } }, "duplicate": { "probabilities": { "false": 0.954006, "true": 0.045994 } }, "matches_order": { "probabilities": { "false": 0.865162, "true": 0.134838 } }, "urgency": { "probabilities": { "0": 0.039431, "1": 0.305, "2": 0.493214, "3": 0.162356 } } }, "invoice_processing_000029": { "discrepancy_severity": { "probabilities": { "0": 0.019486, "1": 0.01863, "2": 0.177175, "3": 0.784709 } }, "disposition": { "probabilities": { "approve": 0.037862, "hold": 0.205164, "manual_review": 0.700978, "reject": 0.055996 } }, "duplicate": { "probabilities": { "false": 0.976511, "true": 0.023489 } }, "matches_order": { "probabilities": { "false": 0.961953, "true": 0.038047 } }, "urgency": { "probabilities": { "0": 0.053753, "1": 0.55998, "2": 0.322299, "3": 0.063968 } } }, "invoice_processing_000030": { "discrepancy_severity": { "probabilities": { "0": 0.786234, "1": 0.048867, "2": 0.052135, "3": 0.112764 } }, "disposition": { "probabilities": { "approve": 0.477728, "hold": 0.097122, "manual_review": 0.070761, "reject": 0.354389 } }, "duplicate": { "probabilities": { "false": 0.41558, "true": 0.58442 } }, "matches_order": { "probabilities": { "false": 0.108791, "true": 0.891209 } }, "urgency": { "probabilities": { "0": 0.07606, "1": 0.680873, "2": 0.19127, "3": 0.051797 } } }, "invoice_processing_000031": { "discrepancy_severity": { "probabilities": { "0": 0.843813, "1": 0.045672, "2": 0.08844, "3": 0.022075 } }, "disposition": { "probabilities": { "approve": 0.769779, "hold": 0.106768, "manual_review": 0.097493, "reject": 0.02596 } }, "duplicate": { "probabilities": { "false": 0.953439, "true": 0.046561 } }, "matches_order": { "probabilities": { "false": 0.104115, "true": 0.895885 } }, "urgency": { "probabilities": { "0": 0.021035, "1": 0.18142, "2": 0.477854, "3": 0.319691 } } }, "invoice_processing_000032": { "discrepancy_severity": { "probabilities": { "0": 0.511112, "1": 0.145659, "2": 0.262392, "3": 0.080837 } }, "disposition": { "probabilities": { "approve": 0.500825, "hold": 0.267464, "manual_review": 0.185049, "reject": 0.046662 } }, "duplicate": { "probabilities": { "false": 0.927939, "true": 0.072061 } }, "matches_order": { "probabilities": { "false": 0.248021, "true": 0.751979 } }, "urgency": { "probabilities": { "0": 0.066088, "1": 0.651044, "2": 0.232839, "3": 0.050029 } } }, "invoice_processing_000033": { "discrepancy_severity": { "probabilities": { "0": 0.298284, "1": 0.074093, "2": 0.361899, "3": 0.265724 } }, "disposition": { "probabilities": { "approve": 0.365867, "hold": 0.207429, "manual_review": 0.385481, "reject": 0.041223 } }, "duplicate": { "probabilities": { "false": 0.977431, "true": 0.022569 } }, "matches_order": { "probabilities": { "false": 0.649125, "true": 0.350875 } }, "urgency": { "probabilities": { "0": 0.04739, "1": 0.659514, "2": 0.254611, "3": 0.038485 } } }, "invoice_processing_000034": { "discrepancy_severity": { "probabilities": { "0": 0.822085, "1": 0.053161, "2": 0.08702, "3": 0.037734 } }, "disposition": { "probabilities": { "approve": 0.681834, "hold": 0.108508, "manual_review": 0.151606, "reject": 0.058052 } }, "duplicate": { "probabilities": { "false": 0.867832, "true": 0.132168 } }, "matches_order": { "probabilities": { "false": 0.131367, "true": 0.868633 } }, "urgency": { "probabilities": { "0": 0.024636, "1": 0.142525, "2": 0.456401, "3": 0.376438 } } }, "invoice_processing_000035": { "discrepancy_severity": { "probabilities": { "0": 0.006106, "1": 0.025975, "2": 0.25423, "3": 0.713689 } }, "disposition": { "probabilities": { "approve": 0.010932, "hold": 0.245683, "manual_review": 0.714277, "reject": 0.029108 } }, "duplicate": { "probabilities": { "false": 0.980334, "true": 0.019666 } }, "matches_order": { "probabilities": { "false": 0.980663, "true": 0.019337 } }, "urgency": { "probabilities": { "0": 0.025567, "1": 0.154254, "2": 0.524448, "3": 0.295731 } } }, "invoice_processing_000036": { "discrepancy_severity": { "probabilities": { "0": 0.889094, "1": 0.038394, "2": 0.056913, "3": 0.015599 } }, "disposition": { "probabilities": { "approve": 0.830871, "hold": 0.085562, "manual_review": 0.067127, "reject": 0.01644 } }, "duplicate": { "probabilities": { "false": 0.96481, "true": 0.03519 } }, "matches_order": { "probabilities": { "false": 0.074044, "true": 0.925956 } }, "urgency": { "probabilities": { "0": 0.019609, "1": 0.150751, "2": 0.455476, "3": 0.374163 } } }, "invoice_processing_000037": { "discrepancy_severity": { "probabilities": { "0": 0.825049, "1": 0.05638, "2": 0.094699, "3": 0.023872 } }, "disposition": { "probabilities": { "approve": 0.75945, "hold": 0.111228, "manual_review": 0.105125, "reject": 0.024197 } }, "duplicate": { "probabilities": { "false": 0.948977, "true": 0.051023 } }, "matches_order": { "probabilities": { "false": 0.110662, "true": 0.889338 } }, "urgency": { "probabilities": { "0": 0.019655, "1": 0.140221, "2": 0.469703, "3": 0.370422 } } }, "invoice_processing_000038": { "discrepancy_severity": { "probabilities": { "0": 0.808761, "1": 0.048987, "2": 0.064684, "3": 0.077568 } }, "disposition": { "probabilities": { "approve": 0.623103, "hold": 0.158533, "manual_review": 0.111041, "reject": 0.107323 } }, "duplicate": { "probabilities": { "false": 0.771472, "true": 0.228528 } }, "matches_order": { "probabilities": { "false": 0.112994, "true": 0.887006 } }, "urgency": { "probabilities": { "0": 0.086064, "1": 0.74084, "2": 0.15175, "3": 0.021346 } } }, "invoice_processing_000039": { "discrepancy_severity": { "probabilities": { "0": 0.030985, "1": 0.062456, "2": 0.5249, "3": 0.381659 } }, "disposition": { "probabilities": { "approve": 0.053967, "hold": 0.260506, "manual_review": 0.637182, "reject": 0.048345 } }, "duplicate": { "probabilities": { "false": 0.971066, "true": 0.028934 } }, "matches_order": { "probabilities": { "false": 0.966182, "true": 0.033818 } }, "urgency": { "probabilities": { "0": 0.041301, "1": 0.553621, "2": 0.335423, "3": 0.069655 } } }, "invoice_processing_000040": { "discrepancy_severity": { "probabilities": { "0": 0.833781, "1": 0.044017, "2": 0.052134, "3": 0.070068 } }, "disposition": { "probabilities": { "approve": 0.673901, "hold": 0.112171, "manual_review": 0.073584, "reject": 0.140344 } }, "duplicate": { "probabilities": { "false": 0.708392, "true": 0.291608 } }, "matches_order": { "probabilities": { "false": 0.090715, "true": 0.909285 } }, "urgency": { "probabilities": { "0": 0.046297, "1": 0.405557, "2": 0.437049, "3": 0.111096 } } }, "invoice_processing_000041": { "discrepancy_severity": { "probabilities": { "0": 0.812473, "1": 0.056207, "2": 0.064116, "3": 0.067204 } }, "disposition": { "probabilities": { "approve": 0.518605, "hold": 0.139787, "manual_review": 0.121914, "reject": 0.219694 } }, "duplicate": { "probabilities": { "false": 0.431391, "true": 0.568609 } }, "matches_order": { "probabilities": { "false": 0.082429, "true": 0.917571 } }, "urgency": { "probabilities": { "0": 0.008699, "1": 0.041481, "2": 0.384872, "3": 0.564948 } } }, "invoice_processing_000042": { "discrepancy_severity": { "probabilities": { "0": 0.803586, "1": 0.050529, "2": 0.079679, "3": 0.066206 } }, "disposition": { "probabilities": { "approve": 0.576063, "hold": 0.138291, "manual_review": 0.155303, "reject": 0.130343 } }, "duplicate": { "probabilities": { "false": 0.689943, "true": 0.310057 } }, "matches_order": { "probabilities": { "false": 0.119155, "true": 0.880845 } }, "urgency": { "probabilities": { "0": 0.019406, "1": 0.15239, "2": 0.440929, "3": 0.387274 } } }, "invoice_processing_000043": { "discrepancy_severity": { "probabilities": { "0": 0.835882, "1": 0.046969, "2": 0.065516, "3": 0.051633 } }, "disposition": { "probabilities": { "approve": 0.639636, "hold": 0.133727, "manual_review": 0.128788, "reject": 0.097849 } }, "duplicate": { "probabilities": { "false": 0.741685, "true": 0.258315 } }, "matches_order": { "probabilities": { "false": 0.094653, "true": 0.905347 } }, "urgency": { "probabilities": { "0": 0.024626, "1": 0.169532, "2": 0.426712, "3": 0.37913 } } }, "invoice_processing_000044": { "discrepancy_severity": { "probabilities": { "0": 0.00567, "1": 0.016271, "2": 0.103487, "3": 0.874572 } }, "disposition": { "probabilities": { "approve": 0.011859, "hold": 0.198925, "manual_review": 0.730188, "reject": 0.059028 } }, "duplicate": { "probabilities": { "false": 0.9651, "true": 0.0349 } }, "matches_order": { "probabilities": { "false": 0.977582, "true": 0.022418 } }, "urgency": { "probabilities": { "0": 0.07995, "1": 0.652877, "2": 0.238669, "3": 0.028504 } } }, "invoice_processing_000045": { "discrepancy_severity": { "probabilities": { "0": 0.254866, "1": 0.096281, "2": 0.455224, "3": 0.193629 } }, "disposition": { "probabilities": { "approve": 0.338332, "hold": 0.247723, "manual_review": 0.385175, "reject": 0.028771 } }, "duplicate": { "probabilities": { "false": 0.974921, "true": 0.025079 } }, "matches_order": { "probabilities": { "false": 0.63063, "true": 0.36937 } }, "urgency": { "probabilities": { "0": 0.03262, "1": 0.385626, "2": 0.487842, "3": 0.093912 } } }, "invoice_processing_000046": { "discrepancy_severity": { "probabilities": { "0": 0.062074, "1": 0.029729, "2": 0.28433, "3": 0.623867 } }, "disposition": { "probabilities": { "approve": 0.063275, "hold": 0.540106, "manual_review": 0.357716, "reject": 0.038903 } }, "duplicate": { "probabilities": { "false": 0.975925, "true": 0.024075 } }, "matches_order": { "probabilities": { "false": 0.905437, "true": 0.094563 } }, "urgency": { "probabilities": { "0": 0.043349, "1": 0.307006, "2": 0.418996, "3": 0.230649 } } }, "invoice_processing_000047": { "discrepancy_severity": { "probabilities": { "0": 0.028153, "1": 0.024533, "2": 0.167076, "3": 0.780238 } }, "disposition": { "probabilities": { "approve": 0.05374, "hold": 0.216363, "manual_review": 0.680462, "reject": 0.049435 } }, "duplicate": { "probabilities": { "false": 0.980742, "true": 0.019258 } }, "matches_order": { "probabilities": { "false": 0.953748, "true": 0.046252 } }, "urgency": { "probabilities": { "0": 0.067917, "1": 0.643767, "2": 0.248211, "3": 0.040105 } } }, "invoice_processing_000048": { "discrepancy_severity": { "probabilities": { "0": 0.374518, "1": 0.17052, "2": 0.384157, "3": 0.070805 } }, "disposition": { "probabilities": { "approve": 0.333428, "hold": 0.344372, "manual_review": 0.284464, "reject": 0.037736 } }, "duplicate": { "probabilities": { "false": 0.940854, "true": 0.059146 } }, "matches_order": { "probabilities": { "false": 0.350965, "true": 0.649035 } }, "urgency": { "probabilities": { "0": 0.02436, "1": 0.148316, "2": 0.493306, "3": 0.334019 } } }, "invoice_processing_000049": { "discrepancy_severity": { "probabilities": { "0": 0.512612, "1": 0.151801, "2": 0.282929, "3": 0.052658 } }, "disposition": { "probabilities": { "approve": 0.446723, "hold": 0.314954, "manual_review": 0.210987, "reject": 0.027337 } }, "duplicate": { "probabilities": { "false": 0.95995, "true": 0.04005 } }, "matches_order": { "probabilities": { "false": 0.224489, "true": 0.775511 } }, "urgency": { "probabilities": { "0": 0.026019, "1": 0.148909, "2": 0.494499, "3": 0.330573 } } }, "invoice_processing_000050": { "discrepancy_severity": { "probabilities": { "0": 0.45295, "1": 0.154358, "2": 0.330767, "3": 0.061926 } }, "disposition": { "probabilities": { "approve": 0.404252, "hold": 0.334588, "manual_review": 0.232166, "reject": 0.028995 } }, "duplicate": { "probabilities": { "false": 0.960157, "true": 0.039843 } }, "matches_order": { "probabilities": { "false": 0.29071, "true": 0.70929 } }, "urgency": { "probabilities": { "0": 0.024028, "1": 0.155502, "2": 0.473696, "3": 0.346774 } } }, "invoice_processing_000051": { "discrepancy_severity": { "probabilities": { "0": 0.052225, "1": 0.017046, "2": 0.08079, "3": 0.849939 } }, "disposition": { "probabilities": { "approve": 0.076714, "hold": 0.14368, "manual_review": 0.629964, "reject": 0.149642 } }, "duplicate": { "probabilities": { "false": 0.970934, "true": 0.029066 } }, "matches_order": { "probabilities": { "false": 0.901369, "true": 0.098631 } }, "urgency": { "probabilities": { "0": 0.023682, "1": 0.191219, "2": 0.492647, "3": 0.292452 } } }, "invoice_processing_000052": { "discrepancy_severity": { "probabilities": { "0": 0.014365, "1": 0.035117, "2": 0.357429, "3": 0.593089 } }, "disposition": { "probabilities": { "approve": 0.014556, "hold": 0.614385, "manual_review": 0.350123, "reject": 0.020936 } }, "duplicate": { "probabilities": { "false": 0.978953, "true": 0.021047 } }, "matches_order": { "probabilities": { "false": 0.96309, "true": 0.03691 } }, "urgency": { "probabilities": { "0": 0.056071, "1": 0.287213, "2": 0.461034, "3": 0.195681 } } }, "invoice_processing_000053": { "discrepancy_severity": { "probabilities": { "0": 0.772325, "1": 0.048765, "2": 0.059016, "3": 0.119894 } }, "disposition": { "probabilities": { "approve": 0.602362, "hold": 0.132478, "manual_review": 0.0903, "reject": 0.17486 } }, "duplicate": { "probabilities": { "false": 0.662073, "true": 0.337927 } }, "matches_order": { "probabilities": { "false": 0.130606, "true": 0.869394 } }, "urgency": { "probabilities": { "0": 0.066818, "1": 0.710713, "2": 0.176139, "3": 0.04633 } } }, "invoice_processing_000054": { "discrepancy_severity": { "probabilities": { "0": 0.003267, "1": 0.021153, "2": 0.262596, "3": 0.712984 } }, "disposition": { "probabilities": { "approve": 0.006909, "hold": 0.265241, "manual_review": 0.702506, "reject": 0.025344 } }, "duplicate": { "probabilities": { "false": 0.977455, "true": 0.022545 } }, "matches_order": { "probabilities": { "false": 0.989966, "true": 0.010034 } }, "urgency": { "probabilities": { "0": 0.042926, "1": 0.29283, "2": 0.461444, "3": 0.2028 } } }, "invoice_processing_000055": { "discrepancy_severity": { "probabilities": { "0": 0.88879, "1": 0.042936, "2": 0.04932, "3": 0.018954 } }, "disposition": { "probabilities": { "approve": 0.837857, "hold": 0.079651, "manual_review": 0.061091, "reject": 0.021401 } }, "duplicate": { "probabilities": { "false": 0.938827, "true": 0.061173 } }, "matches_order": { "probabilities": { "false": 0.0648, "true": 0.9352 } }, "urgency": { "probabilities": { "0": 0.024935, "1": 0.201284, "2": 0.52918, "3": 0.244601 } } }, "invoice_processing_000056": { "discrepancy_severity": { "probabilities": { "0": 0.530561, "1": 0.148747, "2": 0.249798, "3": 0.070894 } }, "disposition": { "probabilities": { "approve": 0.546235, "hold": 0.259582, "manual_review": 0.162174, "reject": 0.032009 } }, "duplicate": { "probabilities": { "false": 0.954483, "true": 0.045517 } }, "matches_order": { "probabilities": { "false": 0.236142, "true": 0.763858 } }, "urgency": { "probabilities": { "0": 0.051896, "1": 0.449843, "2": 0.421474, "3": 0.076788 } } }, "invoice_processing_000057": { "discrepancy_severity": { "probabilities": { "0": 0.887399, "1": 0.036813, "2": 0.056097, "3": 0.019691 } }, "disposition": { "probabilities": { "approve": 0.846164, "hold": 0.086566, "manual_review": 0.048676, "reject": 0.018594 } }, "duplicate": { "probabilities": { "false": 0.962843, "true": 0.037157 } }, "matches_order": { "probabilities": { "false": 0.077436, "true": 0.922564 } }, "urgency": { "probabilities": { "0": 0.037212, "1": 0.383954, "2": 0.427474, "3": 0.15136 } } }, "invoice_processing_000058": { "discrepancy_severity": { "probabilities": { "0": 0.020151, "1": 0.01909, "2": 0.120494, "3": 0.840265 } }, "disposition": { "probabilities": { "approve": 0.036578, "hold": 0.207459, "manual_review": 0.699249, "reject": 0.056714 } }, "duplicate": { "probabilities": { "false": 0.97846, "true": 0.02154 } }, "matches_order": { "probabilities": { "false": 0.955932, "true": 0.044068 } }, "urgency": { "probabilities": { "0": 0.076351, "1": 0.501507, "2": 0.365319, "3": 0.056823 } } }, "invoice_processing_000059": { "discrepancy_severity": { "probabilities": { "0": 0.118984, "1": 0.036736, "2": 0.24162, "3": 0.60266 } }, "disposition": { "probabilities": { "approve": 0.108887, "hold": 0.348903, "manual_review": 0.473347, "reject": 0.068864 } }, "duplicate": { "probabilities": { "false": 0.939174, "true": 0.060826 } }, "matches_order": { "probabilities": { "false": 0.792612, "true": 0.207388 } }, "urgency": { "probabilities": { "0": 0.017357, "1": 0.094773, "2": 0.470895, "3": 0.416976 } } }, "invoice_processing_000060": { "discrepancy_severity": { "probabilities": { "0": 0.317306, "1": 0.0672, "2": 0.304683, "3": 0.310811 } }, "disposition": { "probabilities": { "approve": 0.380966, "hold": 0.206276, "manual_review": 0.372665, "reject": 0.040092 } }, "duplicate": { "probabilities": { "false": 0.979652, "true": 0.020348 } }, "matches_order": { "probabilities": { "false": 0.638644, "true": 0.361356 } }, "urgency": { "probabilities": { "0": 0.04806, "1": 0.625351, "2": 0.276093, "3": 0.050496 } } }, "invoice_processing_000061": { "discrepancy_severity": { "probabilities": { "0": 0.112731, "1": 0.103044, "2": 0.373171, "3": 0.411055 } }, "disposition": { "probabilities": { "approve": 0.130278, "hold": 0.24283, "manual_review": 0.586404, "reject": 0.040488 } }, "duplicate": { "probabilities": { "false": 0.957254, "true": 0.042746 } }, "matches_order": { "probabilities": { "false": 0.745956, "true": 0.254044 } }, "urgency": { "probabilities": { "0": 0.025513, "1": 0.156818, "2": 0.517168, "3": 0.300501 } } }, "invoice_processing_000062": { "discrepancy_severity": { "probabilities": { "0": 0.77535, "1": 0.054841, "2": 0.077104, "3": 0.092705 } }, "disposition": { "probabilities": { "approve": 0.589554, "hold": 0.169906, "manual_review": 0.116469, "reject": 0.124071 } }, "duplicate": { "probabilities": { "false": 0.771805, "true": 0.228195 } }, "matches_order": { "probabilities": { "false": 0.133326, "true": 0.866674 } }, "urgency": { "probabilities": { "0": 0.06692, "1": 0.614335, "2": 0.260761, "3": 0.057984 } } }, "invoice_processing_000063": { "discrepancy_severity": { "probabilities": { "0": 0.872043, "1": 0.040877, "2": 0.051432, "3": 0.035648 } }, "disposition": { "probabilities": { "approve": 0.840438, "hold": 0.069938, "manual_review": 0.053893, "reject": 0.035731 } }, "duplicate": { "probabilities": { "false": 0.927888, "true": 0.072112 } }, "matches_order": { "probabilities": { "false": 0.089718, "true": 0.910282 } }, "urgency": { "probabilities": { "0": 0.055647, "1": 0.646469, "2": 0.232176, "3": 0.065708 } } }, "invoice_processing_000064": { "discrepancy_severity": { "probabilities": { "0": 0.829853, "1": 0.047281, "2": 0.053264, "3": 0.069602 } }, "disposition": { "probabilities": { "approve": 0.669112, "hold": 0.137568, "manual_review": 0.07799, "reject": 0.11533 } }, "duplicate": { "probabilities": { "false": 0.747256, "true": 0.252744 } }, "matches_order": { "probabilities": { "false": 0.08959, "true": 0.91041 } }, "urgency": { "probabilities": { "0": 0.022012, "1": 0.156491, "2": 0.584774, "3": 0.236723 } } }, "invoice_processing_000065": { "discrepancy_severity": { "probabilities": { "0": 0.468678, "1": 0.156585, "2": 0.325081, "3": 0.049656 } }, "disposition": { "probabilities": { "approve": 0.439793, "hold": 0.321445, "manual_review": 0.21111, "reject": 0.027652 } }, "duplicate": { "probabilities": { "false": 0.960283, "true": 0.039717 } }, "matches_order": { "probabilities": { "false": 0.231253, "true": 0.768747 } }, "urgency": { "probabilities": { "0": 0.017786, "1": 0.108831, "2": 0.520398, "3": 0.352985 } } }, "invoice_processing_000066": { "discrepancy_severity": { "probabilities": { "0": 0.448533, "1": 0.149319, "2": 0.338828, "3": 0.063321 } }, "disposition": { "probabilities": { "approve": 0.397774, "hold": 0.33729, "manual_review": 0.23531, "reject": 0.029626 } }, "duplicate": { "probabilities": { "false": 0.959033, "true": 0.040967 } }, "matches_order": { "probabilities": { "false": 0.289391, "true": 0.710609 } }, "urgency": { "probabilities": { "0": 0.024178, "1": 0.151943, "2": 0.461894, "3": 0.361985 } } }, "invoice_processing_000067": { "discrepancy_severity": { "probabilities": { "0": 0.868824, "1": 0.044457, "2": 0.070467, "3": 0.016252 } }, "disposition": { "probabilities": { "approve": 0.812534, "hold": 0.089722, "manual_review": 0.076408, "reject": 0.021336 } }, "duplicate": { "probabilities": { "false": 0.949873, "true": 0.050127 } }, "matches_order": { "probabilities": { "false": 0.078377, "true": 0.921623 } }, "urgency": { "probabilities": { "0": 0.022248, "1": 0.171461, "2": 0.477015, "3": 0.329276 } } }, "invoice_processing_000068": { "discrepancy_severity": { "probabilities": { "0": 0.062881, "1": 0.160143, "2": 0.661571, "3": 0.115405 } }, "disposition": { "probabilities": { "approve": 0.122508, "hold": 0.340244, "manual_review": 0.513441, "reject": 0.023807 } }, "duplicate": { "probabilities": { "false": 0.939736, "true": 0.060264 } }, "matches_order": { "probabilities": { "false": 0.813349, "true": 0.186651 } }, "urgency": { "probabilities": { "0": 0.052875, "1": 0.45046, "2": 0.423673, "3": 0.072992 } } }, "invoice_processing_000069": { "discrepancy_severity": { "probabilities": { "0": 0.867059, "1": 0.043512, "2": 0.070773, "3": 0.018656 } }, "disposition": { "probabilities": { "approve": 0.816381, "hold": 0.088239, "manual_review": 0.070399, "reject": 0.024981 } }, "duplicate": { "probabilities": { "false": 0.951827, "true": 0.048173 } }, "matches_order": { "probabilities": { "false": 0.084742, "true": 0.915258 } }, "urgency": { "probabilities": { "0": 0.011615, "1": 0.103355, "2": 0.458607, "3": 0.426423 } } }, "invoice_processing_000070": { "discrepancy_severity": { "probabilities": { "0": 0.00685, "1": 0.025441, "2": 0.201015, "3": 0.766694 } }, "disposition": { "probabilities": { "approve": 0.013803, "hold": 0.281393, "manual_review": 0.665903, "reject": 0.038901 } }, "duplicate": { "probabilities": { "false": 0.981387, "true": 0.018613 } }, "matches_order": { "probabilities": { "false": 0.980873, "true": 0.019127 } }, "urgency": { "probabilities": { "0": 0.070963, "1": 0.632883, "2": 0.26118, "3": 0.034974 } } }, "invoice_processing_000071": { "discrepancy_severity": { "probabilities": { "0": 0.410604, "1": 0.136096, "2": 0.338545, "3": 0.114755 } }, "disposition": { "probabilities": { "approve": 0.40794, "hold": 0.377609, "manual_review": 0.178709, "reject": 0.035742 } }, "duplicate": { "probabilities": { "false": 0.97302, "true": 0.02698 } }, "matches_order": { "probabilities": { "false": 0.332939, "true": 0.667061 } }, "urgency": { "probabilities": { "0": 0.056464, "1": 0.474908, "2": 0.404335, "3": 0.064294 } } }, "invoice_processing_000072": { "discrepancy_severity": { "probabilities": { "0": 0.079169, "1": 0.037152, "2": 0.363589, "3": 0.52009 } }, "disposition": { "probabilities": { "approve": 0.081933, "hold": 0.530955, "manual_review": 0.350592, "reject": 0.03652 } }, "duplicate": { "probabilities": { "false": 0.977833, "true": 0.022167 } }, "matches_order": { "probabilities": { "false": 0.886079, "true": 0.113921 } }, "urgency": { "probabilities": { "0": 0.067693, "1": 0.597074, "2": 0.269897, "3": 0.065336 } } }, "invoice_processing_000073": { "discrepancy_severity": { "probabilities": { "0": 0.003202, "1": 0.018655, "2": 0.169051, "3": 0.809092 } }, "disposition": { "probabilities": { "approve": 0.006187, "hold": 0.254604, "manual_review": 0.716318, "reject": 0.022891 } }, "duplicate": { "probabilities": { "false": 0.982983, "true": 0.017017 } }, "matches_order": { "probabilities": { "false": 0.988744, "true": 0.011256 } }, "urgency": { "probabilities": { "0": 0.097437, "1": 0.517035, "2": 0.339056, "3": 0.046472 } } }, "invoice_processing_000074": { "discrepancy_severity": { "probabilities": { "0": 0.40674, "1": 0.045867, "2": 0.157027, "3": 0.390365 } }, "disposition": { "probabilities": { "approve": 0.405466, "hold": 0.169422, "manual_review": 0.364907, "reject": 0.060205 } }, "duplicate": { "probabilities": { "false": 0.980437, "true": 0.019563 } }, "matches_order": { "probabilities": { "false": 0.481886, "true": 0.518114 } }, "urgency": { "probabilities": { "0": 0.028762, "1": 0.220144, "2": 0.509335, "3": 0.241759 } } }, "invoice_processing_000075": { "discrepancy_severity": { "probabilities": { "0": 0.873539, "1": 0.041069, "2": 0.061003, "3": 0.024389 } }, "disposition": { "probabilities": { "approve": 0.854809, "hold": 0.080687, "manual_review": 0.044205, "reject": 0.020299 } }, "duplicate": { "probabilities": { "false": 0.968529, "true": 0.031471 } }, "matches_order": { "probabilities": { "false": 0.096643, "true": 0.903357 } }, "urgency": { "probabilities": { "0": 0.048776, "1": 0.492442, "2": 0.376863, "3": 0.081919 } } }, "invoice_processing_000076": { "discrepancy_severity": { "probabilities": { "0": 0.011938, "1": 0.048835, "2": 0.488514, "3": 0.450712 } }, "disposition": { "probabilities": { "approve": 0.014869, "hold": 0.268165, "manual_review": 0.669386, "reject": 0.04758 } }, "duplicate": { "probabilities": { "false": 0.970674, "true": 0.029326 } }, "matches_order": { "probabilities": { "false": 0.970229, "true": 0.029771 } }, "urgency": { "probabilities": { "0": 0.035412, "1": 0.277925, "2": 0.520253, "3": 0.16641 } } }, "invoice_processing_000077": { "discrepancy_severity": { "probabilities": { "0": 0.007797, "1": 0.036459, "2": 0.294694, "3": 0.66105 } }, "disposition": { "probabilities": { "approve": 0.012864, "hold": 0.25161, "manual_review": 0.707306, "reject": 0.02822 } }, "duplicate": { "probabilities": { "false": 0.974607, "true": 0.025393 } }, "matches_order": { "probabilities": { "false": 0.983054, "true": 0.016946 } }, "urgency": { "probabilities": { "0": 0.045437, "1": 0.285716, "2": 0.46031, "3": 0.208537 } } }, "invoice_processing_000078": { "discrepancy_severity": { "probabilities": { "0": 0.005317, "1": 0.037993, "2": 0.436416, "3": 0.520274 } }, "disposition": { "probabilities": { "approve": 0.009786, "hold": 0.254474, "manual_review": 0.708688, "reject": 0.027052 } }, "duplicate": { "probabilities": { "false": 0.966162, "true": 0.033838 } }, "matches_order": { "probabilities": { "false": 0.985667, "true": 0.014333 } }, "urgency": { "probabilities": { "0": 0.035099, "1": 0.261167, "2": 0.490142, "3": 0.213593 } } }, "invoice_processing_000079": { "discrepancy_severity": { "probabilities": { "0": 0.069868, "1": 0.019954, "2": 0.08233, "3": 0.827848 } }, "disposition": { "probabilities": { "approve": 0.103276, "hold": 0.150852, "manual_review": 0.610001, "reject": 0.135871 } }, "duplicate": { "probabilities": { "false": 0.974163, "true": 0.025837 } }, "matches_order": { "probabilities": { "false": 0.864099, "true": 0.135901 } }, "urgency": { "probabilities": { "0": 0.023333, "1": 0.191647, "2": 0.509833, "3": 0.275187 } } }, "invoice_processing_000080": { "discrepancy_severity": { "probabilities": { "0": 0.058539, "1": 0.047119, "2": 0.242808, "3": 0.651534 } }, "disposition": { "probabilities": { "approve": 0.089617, "hold": 0.210043, "manual_review": 0.638929, "reject": 0.061411 } }, "duplicate": { "probabilities": { "false": 0.960674, "true": 0.039326 } }, "matches_order": { "probabilities": { "false": 0.905362, "true": 0.094638 } }, "urgency": { "probabilities": { "0": 0.018733, "1": 0.147559, "2": 0.656074, "3": 0.177634 } } }, "invoice_processing_000081": { "discrepancy_severity": { "probabilities": { "0": 0.835331, "1": 0.047025, "2": 0.082581, "3": 0.035063 } }, "disposition": { "probabilities": { "approve": 0.785998, "hold": 0.099178, "manual_review": 0.085745, "reject": 0.029079 } }, "duplicate": { "probabilities": { "false": 0.942935, "true": 0.057065 } }, "matches_order": { "probabilities": { "false": 0.124764, "true": 0.875236 } }, "urgency": { "probabilities": { "0": 0.049203, "1": 0.45998, "2": 0.417993, "3": 0.072824 } } }, "invoice_processing_000082": { "discrepancy_severity": { "probabilities": { "0": 0.838852, "1": 0.045928, "2": 0.073325, "3": 0.041895 } }, "disposition": { "probabilities": { "approve": 0.770381, "hold": 0.10146, "manual_review": 0.097044, "reject": 0.031115 } }, "duplicate": { "probabilities": { "false": 0.927787, "true": 0.072213 } }, "matches_order": { "probabilities": { "false": 0.12485, "true": 0.87515 } }, "urgency": { "probabilities": { "0": 0.079258, "1": 0.72114, "2": 0.171968, "3": 0.027634 } } }, "invoice_processing_000083": { "discrepancy_severity": { "probabilities": { "0": 0.006384, "1": 0.03566, "2": 0.338722, "3": 0.619234 } }, "disposition": { "probabilities": { "approve": 0.010071, "hold": 0.229572, "manual_review": 0.723048, "reject": 0.037309 } }, "duplicate": { "probabilities": { "false": 0.964378, "true": 0.035622 } }, "matches_order": { "probabilities": { "false": 0.981509, "true": 0.018491 } }, "urgency": { "probabilities": { "0": 0.022456, "1": 0.195831, "2": 0.582705, "3": 0.199008 } } }, "invoice_processing_000084": { "discrepancy_severity": { "probabilities": { "0": 0.016852, "1": 0.033409, "2": 0.320382, "3": 0.629357 } }, "disposition": { "probabilities": { "approve": 0.019718, "hold": 0.516047, "manual_review": 0.431466, "reject": 0.032769 } }, "duplicate": { "probabilities": { "false": 0.955724, "true": 0.044276 } }, "matches_order": { "probabilities": { "false": 0.947968, "true": 0.052032 } }, "urgency": { "probabilities": { "0": 0.087046, "1": 0.56073, "2": 0.300738, "3": 0.051486 } } }, "invoice_processing_000085": { "discrepancy_severity": { "probabilities": { "0": 0.023335, "1": 0.031907, "2": 0.432522, "3": 0.512236 } }, "disposition": { "probabilities": { "approve": 0.028998, "hold": 0.216182, "manual_review": 0.696435, "reject": 0.058385 } }, "duplicate": { "probabilities": { "false": 0.974206, "true": 0.025794 } }, "matches_order": { "probabilities": { "false": 0.965998, "true": 0.034002 } }, "urgency": { "probabilities": { "0": 0.032399, "1": 0.312065, "2": 0.464611, "3": 0.190926 } } }, "invoice_processing_000086": { "discrepancy_severity": { "probabilities": { "0": 0.8721, "1": 0.040966, "2": 0.064207, "3": 0.022727 } }, "disposition": { "probabilities": { "approve": 0.84055, "hold": 0.079725, "manual_review": 0.05407, "reject": 0.025655 } }, "duplicate": { "probabilities": { "false": 0.956056, "true": 0.043944 } }, "matches_order": { "probabilities": { "false": 0.081773, "true": 0.918227 } }, "urgency": { "probabilities": { "0": 0.037083, "1": 0.4063, "2": 0.41738, "3": 0.139237 } } }, "invoice_processing_000087": { "discrepancy_severity": { "probabilities": { "0": 0.054222, "1": 0.034055, "2": 0.213262, "3": 0.698461 } }, "disposition": { "probabilities": { "approve": 0.084084, "hold": 0.244625, "manual_review": 0.604984, "reject": 0.066307 } }, "duplicate": { "probabilities": { "false": 0.980565, "true": 0.019435 } }, "matches_order": { "probabilities": { "false": 0.927543, "true": 0.072457 } }, "urgency": { "probabilities": { "0": 0.04661, "1": 0.418648, "2": 0.449062, "3": 0.08568 } } }, "invoice_processing_000088": { "discrepancy_severity": { "probabilities": { "0": 0.476345, "1": 0.166046, "2": 0.314885, "3": 0.042724 } }, "disposition": { "probabilities": { "approve": 0.455014, "hold": 0.291941, "manual_review": 0.223132, "reject": 0.029914 } }, "duplicate": { "probabilities": { "false": 0.939948, "true": 0.060052 } }, "matches_order": { "probabilities": { "false": 0.204911, "true": 0.795089 } }, "urgency": { "probabilities": { "0": 0.014873, "1": 0.083494, "2": 0.501819, "3": 0.399814 } } }, "invoice_processing_000089": { "discrepancy_severity": { "probabilities": { "0": 0.895939, "1": 0.032788, "2": 0.053467, "3": 0.017806 } }, "disposition": { "probabilities": { "approve": 0.855536, "hold": 0.074608, "manual_review": 0.049294, "reject": 0.020562 } }, "duplicate": { "probabilities": { "false": 0.961655, "true": 0.038345 } }, "matches_order": { "probabilities": { "false": 0.081393, "true": 0.918607 } }, "urgency": { "probabilities": { "0": 0.035622, "1": 0.38851, "2": 0.390645, "3": 0.185223 } } }, "invoice_processing_000090": { "discrepancy_severity": { "probabilities": { "0": 0.031135, "1": 0.024421, "2": 0.19393, "3": 0.750514 } }, "disposition": { "probabilities": { "approve": 0.05141, "hold": 0.162782, "manual_review": 0.709593, "reject": 0.076215 } }, "duplicate": { "probabilities": { "false": 0.958769, "true": 0.041231 } }, "matches_order": { "probabilities": { "false": 0.941498, "true": 0.058502 } }, "urgency": { "probabilities": { "0": 0.022232, "1": 0.200377, "2": 0.468679, "3": 0.308712 } } }, "invoice_processing_000091": { "discrepancy_severity": { "probabilities": { "0": 0.836336, "1": 0.039244, "2": 0.044528, "3": 0.079892 } }, "disposition": { "probabilities": { "approve": 0.636279, "hold": 0.096787, "manual_review": 0.09154, "reject": 0.175394 } }, "duplicate": { "probabilities": { "false": 0.556324, "true": 0.443676 } }, "matches_order": { "probabilities": { "false": 0.097212, "true": 0.902788 } }, "urgency": { "probabilities": { "0": 0.03974, "1": 0.347197, "2": 0.340417, "3": 0.272646 } } }, "invoice_processing_000092": { "discrepancy_severity": { "probabilities": { "0": 0.327229, "1": 0.196148, "2": 0.286416, "3": 0.190208 } }, "disposition": { "probabilities": { "approve": 0.189301, "hold": 0.352899, "manual_review": 0.27475, "reject": 0.183051 } }, "duplicate": { "probabilities": { "false": 0.529624, "true": 0.470376 } }, "matches_order": { "probabilities": { "false": 0.309229, "true": 0.690771 } }, "urgency": { "probabilities": { "0": 0.018072, "1": 0.074511, "2": 0.441261, "3": 0.466156 } } }, "invoice_processing_000093": { "discrepancy_severity": { "probabilities": { "0": 0.718062, "1": 0.055484, "2": 0.097928, "3": 0.128526 } }, "disposition": { "probabilities": { "approve": 0.561866, "hold": 0.182977, "manual_review": 0.124732, "reject": 0.130425 } }, "duplicate": { "probabilities": { "false": 0.749462, "true": 0.250538 } }, "matches_order": { "probabilities": { "false": 0.192849, "true": 0.807151 } }, "urgency": { "probabilities": { "0": 0.073024, "1": 0.777084, "2": 0.125516, "3": 0.024376 } } }, "invoice_processing_000094": { "discrepancy_severity": { "probabilities": { "0": 0.492162, "1": 0.144901, "2": 0.298572, "3": 0.064365 } }, "disposition": { "probabilities": { "approve": 0.466826, "hold": 0.316417, "manual_review": 0.183321, "reject": 0.033436 } }, "duplicate": { "probabilities": { "false": 0.956554, "true": 0.043446 } }, "matches_order": { "probabilities": { "false": 0.26536, "true": 0.73464 } }, "urgency": { "probabilities": { "0": 0.041424, "1": 0.329265, "2": 0.437182, "3": 0.192128 } } }, "invoice_processing_000095": { "discrepancy_severity": { "probabilities": { "0": 0.036733, "1": 0.032808, "2": 0.251191, "3": 0.679268 } }, "disposition": { "probabilities": { "approve": 0.050656, "hold": 0.14763, "manual_review": 0.731932, "reject": 0.069782 } }, "duplicate": { "probabilities": { "false": 0.956635, "true": 0.043365 } }, "matches_order": { "probabilities": { "false": 0.946004, "true": 0.053996 } }, "urgency": { "probabilities": { "0": 0.018432, "1": 0.185228, "2": 0.508961, "3": 0.287379 } } }, "invoice_processing_000096": { "discrepancy_severity": { "probabilities": { "0": 0.427962, "1": 0.173814, "2": 0.341972, "3": 0.056252 } }, "disposition": { "probabilities": { "approve": 0.401477, "hold": 0.327082, "manual_review": 0.241659, "reject": 0.029782 } }, "duplicate": { "probabilities": { "false": 0.944046, "true": 0.055954 } }, "matches_order": { "probabilities": { "false": 0.264942, "true": 0.735058 } }, "urgency": { "probabilities": { "0": 0.024405, "1": 0.13731, "2": 0.489675, "3": 0.348609 } } }, "invoice_processing_000097": { "discrepancy_severity": { "probabilities": { "0": 0.884513, "1": 0.03556, "2": 0.057593, "3": 0.022334 } }, "disposition": { "probabilities": { "approve": 0.860377, "hold": 0.075445, "manual_review": 0.044288, "reject": 0.01989 } }, "duplicate": { "probabilities": { "false": 0.96522, "true": 0.03478 } }, "matches_order": { "probabilities": { "false": 0.089623, "true": 0.910377 } }, "urgency": { "probabilities": { "0": 0.054751, "1": 0.680252, "2": 0.208198, "3": 0.056799 } } }, "invoice_processing_000098": { "discrepancy_severity": { "probabilities": { "0": 0.471154, "1": 0.138367, "2": 0.309029, "3": 0.08145 } }, "disposition": { "probabilities": { "approve": 0.471358, "hold": 0.328625, "manual_review": 0.165041, "reject": 0.034976 } }, "duplicate": { "probabilities": { "false": 0.962973, "true": 0.037027 } }, "matches_order": { "probabilities": { "false": 0.292629, "true": 0.707371 } }, "urgency": { "probabilities": { "0": 0.067774, "1": 0.711865, "2": 0.18804, "3": 0.032321 } } }, "invoice_processing_000099": { "discrepancy_severity": { "probabilities": { "0": 0.46257, "1": 0.145867, "2": 0.304003, "3": 0.087561 } }, "disposition": { "probabilities": { "approve": 0.436606, "hold": 0.352673, "manual_review": 0.18207, "reject": 0.028652 } }, "duplicate": { "probabilities": { "false": 0.973526, "true": 0.026474 } }, "matches_order": { "probabilities": { "false": 0.314779, "true": 0.685221 } }, "urgency": { "probabilities": { "0": 0.038257, "1": 0.32539, "2": 0.44657, "3": 0.189783 } } }, "security_incidents_000000": { "credential_compromise": { "probabilities": { "false": 0.331234, "true": 0.668766 } }, "disposition": { "probabilities": { "close_benign": 0.018686, "contain": 0.330052, "investigate": 0.564218, "monitor": 0.087044 } }, "severity": { "probabilities": { "0": 0.033145, "1": 0.13884, "2": 0.308414, "3": 0.433033, "4": 0.086568 } }, "true_positive": { "probabilities": { "false": 0.158658, "true": 0.841342 } }, "urgency": { "probabilities": { "0": 0.014775, "1": 0.080362, "2": 0.29646, "3": 0.608403 } } }, "security_incidents_000001": { "credential_compromise": { "probabilities": { "false": 0.308583, "true": 0.691417 } }, "disposition": { "probabilities": { "close_benign": 0.018241, "contain": 0.285393, "investigate": 0.597589, "monitor": 0.098777 } }, "severity": { "probabilities": { "0": 0.027123, "1": 0.139602, "2": 0.304119, "3": 0.430585, "4": 0.09857 } }, "true_positive": { "probabilities": { "false": 0.179693, "true": 0.820307 } }, "urgency": { "probabilities": { "0": 0.014223, "1": 0.090172, "2": 0.318304, "3": 0.577301 } } }, "security_incidents_000002": { "credential_compromise": { "probabilities": { "false": 0.764128, "true": 0.235872 } }, "disposition": { "probabilities": { "close_benign": 0.351428, "contain": 0.054223, "investigate": 0.317072, "monitor": 0.277277 } }, "severity": { "probabilities": { "0": 0.115779, "1": 0.220885, "2": 0.260071, "3": 0.340274, "4": 0.062992 } }, "true_positive": { "probabilities": { "false": 0.644553, "true": 0.355447 } }, "urgency": { "probabilities": { "0": 0.192972, "1": 0.37522, "2": 0.287384, "3": 0.144424 } } }, "security_incidents_000003": { "credential_compromise": { "probabilities": { "false": 0.286889, "true": 0.713111 } }, "disposition": { "probabilities": { "close_benign": 0.047066, "contain": 0.279154, "investigate": 0.556691, "monitor": 0.117089 } }, "severity": { "probabilities": { "0": 0.014382, "1": 0.079816, "2": 0.400548, "3": 0.431684, "4": 0.07357 } }, "true_positive": { "probabilities": { "false": 0.22814, "true": 0.77186 } }, "urgency": { "probabilities": { "0": 0.021985, "1": 0.093994, "2": 0.290841, "3": 0.59318 } } }, "security_incidents_000004": { "credential_compromise": { "probabilities": { "false": 0.604863, "true": 0.395137 } }, "disposition": { "probabilities": { "close_benign": 0.133908, "contain": 0.065818, "investigate": 0.54909, "monitor": 0.251184 } }, "severity": { "probabilities": { "0": 0.042148, "1": 0.191758, "2": 0.529227, "3": 0.213174, "4": 0.023693 } }, "true_positive": { "probabilities": { "false": 0.543173, "true": 0.456827 } }, "urgency": { "probabilities": { "0": 0.069056, "1": 0.361048, "2": 0.386974, "3": 0.182921 } } }, "security_incidents_000005": { "credential_compromise": { "probabilities": { "false": 0.300256, "true": 0.699744 } }, "disposition": { "probabilities": { "close_benign": 0.025204, "contain": 0.392633, "investigate": 0.502656, "monitor": 0.079507 } }, "severity": { "probabilities": { "0": 0.002799, "1": 0.021341, "2": 0.112774, "3": 0.385863, "4": 0.477223 } }, "true_positive": { "probabilities": { "false": 0.198986, "true": 0.801014 } }, "urgency": { "probabilities": { "0": 0.006184, "1": 0.047596, "2": 0.161507, "3": 0.784713 } } }, "security_incidents_000006": { "credential_compromise": { "probabilities": { "false": 0.303729, "true": 0.696271 } }, "disposition": { "probabilities": { "close_benign": 0.026901, "contain": 0.281278, "investigate": 0.592033, "monitor": 0.099788 } }, "severity": { "probabilities": { "0": 0.009654, "1": 0.073828, "2": 0.277417, "3": 0.457477, "4": 0.181625 } }, "true_positive": { "probabilities": { "false": 0.192665, "true": 0.807335 } }, "urgency": { "probabilities": { "0": 0.010417, "1": 0.066182, "2": 0.259486, "3": 0.663915 } } }, "security_incidents_000007": { "credential_compromise": { "probabilities": { "false": 0.334444, "true": 0.665556 } }, "disposition": { "probabilities": { "close_benign": 0.021657, "contain": 0.378837, "investigate": 0.527446, "monitor": 0.07206 } }, "severity": { "probabilities": { "0": 0.014687, "1": 0.093204, "2": 0.449611, "3": 0.353815, "4": 0.088683 } }, "true_positive": { "probabilities": { "false": 0.143025, "true": 0.856975 } }, "urgency": { "probabilities": { "0": 0.010753, "1": 0.075736, "2": 0.278886, "3": 0.634625 } } }, "security_incidents_000008": { "credential_compromise": { "probabilities": { "false": 0.370729, "true": 0.629271 } }, "disposition": { "probabilities": { "close_benign": 0.025456, "contain": 0.207403, "investigate": 0.632565, "monitor": 0.134576 } }, "severity": { "probabilities": { "0": 0.017088, "1": 0.101716, "2": 0.244589, "3": 0.494422, "4": 0.142185 } }, "true_positive": { "probabilities": { "false": 0.240578, "true": 0.759422 } }, "urgency": { "probabilities": { "0": 0.014798, "1": 0.11662, "2": 0.316738, "3": 0.551844 } } }, "security_incidents_000009": { "credential_compromise": { "probabilities": { "false": 0.648685, "true": 0.351315 } }, "disposition": { "probabilities": { "close_benign": 0.320042, "contain": 0.046598, "investigate": 0.356128, "monitor": 0.277232 } }, "severity": { "probabilities": { "0": 0.07621, "1": 0.241633, "2": 0.45068, "3": 0.204356, "4": 0.027121 } }, "true_positive": { "probabilities": { "false": 0.6071, "true": 0.3929 } }, "urgency": { "probabilities": { "0": 0.169817, "1": 0.37956, "2": 0.32328, "3": 0.127343 } } }, "security_incidents_000010": { "credential_compromise": { "probabilities": { "false": 0.475321, "true": 0.524679 } }, "disposition": { "probabilities": { "close_benign": 0.058871, "contain": 0.14399, "investigate": 0.586306, "monitor": 0.210833 } }, "severity": { "probabilities": { "0": 0.016657, "1": 0.095508, "2": 0.230977, "3": 0.535785, "4": 0.121073 } }, "true_positive": { "probabilities": { "false": 0.468077, "true": 0.531923 } }, "urgency": { "probabilities": { "0": 0.034926, "1": 0.200329, "2": 0.393551, "3": 0.371194 } } }, "security_incidents_000011": { "credential_compromise": { "probabilities": { "false": 0.419371, "true": 0.580629 } }, "disposition": { "probabilities": { "close_benign": 0.046533, "contain": 0.267302, "investigate": 0.545022, "monitor": 0.141143 } }, "severity": { "probabilities": { "0": 0.010836, "1": 0.060723, "2": 0.1574, "3": 0.320998, "4": 0.450043 } }, "true_positive": { "probabilities": { "false": 0.378086, "true": 0.621914 } }, "urgency": { "probabilities": { "0": 0.024067, "1": 0.14002, "2": 0.263904, "3": 0.572009 } } }, "security_incidents_000012": { "credential_compromise": { "probabilities": { "false": 0.548532, "true": 0.451468 } }, "disposition": { "probabilities": { "close_benign": 0.124378, "contain": 0.059826, "investigate": 0.473144, "monitor": 0.342652 } }, "severity": { "probabilities": { "0": 0.201923, "1": 0.468022, "2": 0.251252, "3": 0.070431, "4": 0.008372 } }, "true_positive": { "probabilities": { "false": 0.484786, "true": 0.515214 } }, "urgency": { "probabilities": { "0": 0.105692, "1": 0.469604, "2": 0.320544, "3": 0.104159 } } }, "security_incidents_000013": { "credential_compromise": { "probabilities": { "false": 0.739765, "true": 0.260235 } }, "disposition": { "probabilities": { "close_benign": 0.277402, "contain": 0.035542, "investigate": 0.397705, "monitor": 0.289352 } }, "severity": { "probabilities": { "0": 0.096013, "1": 0.267805, "2": 0.491887, "3": 0.13003, "4": 0.014264 } }, "true_positive": { "probabilities": { "false": 0.640864, "true": 0.359136 } }, "urgency": { "probabilities": { "0": 0.160561, "1": 0.389347, "2": 0.345536, "3": 0.104557 } } }, "security_incidents_000014": { "credential_compromise": { "probabilities": { "false": 0.205716, "true": 0.794284 } }, "disposition": { "probabilities": { "close_benign": 0.022845, "contain": 0.549092, "investigate": 0.369662, "monitor": 0.058401 } }, "severity": { "probabilities": { "0": 0.004962, "1": 0.021178, "2": 0.123606, "3": 0.55713, "4": 0.293124 } }, "true_positive": { "probabilities": { "false": 0.14783, "true": 0.85217 } }, "urgency": { "probabilities": { "0": 0.008926, "1": 0.039452, "2": 0.140499, "3": 0.811123 } } }, "security_incidents_000015": { "credential_compromise": { "probabilities": { "false": 0.414322, "true": 0.585678 } }, "disposition": { "probabilities": { "close_benign": 0.036606, "contain": 0.135501, "investigate": 0.659658, "monitor": 0.168235 } }, "severity": { "probabilities": { "0": 0.059501, "1": 0.301118, "2": 0.399915, "3": 0.205271, "4": 0.034195 } }, "true_positive": { "probabilities": { "false": 0.274802, "true": 0.725198 } }, "urgency": { "probabilities": { "0": 0.031604, "1": 0.216092, "2": 0.432772, "3": 0.319532 } } }, "security_incidents_000020": { "credential_compromise": { "probabilities": { "false": 0.518481, "true": 0.481519 } }, "disposition": { "probabilities": { "close_benign": 0.092745, "contain": 0.12752, "investigate": 0.616701, "monitor": 0.163034 } }, "severity": { "probabilities": { "0": 0.02468, "1": 0.123508, "2": 0.492834, "3": 0.30306, "4": 0.055918 } }, "true_positive": { "probabilities": { "false": 0.341701, "true": 0.658299 } }, "urgency": { "probabilities": { "0": 0.048088, "1": 0.179391, "2": 0.403706, "3": 0.368815 } } }, "security_incidents_000021": { "credential_compromise": { "probabilities": { "false": 0.378154, "true": 0.621846 } }, "disposition": { "probabilities": { "close_benign": 0.078865, "contain": 0.160207, "investigate": 0.601572, "monitor": 0.159356 } }, "severity": { "probabilities": { "0": 0.005559, "1": 0.027775, "2": 0.206869, "3": 0.635265, "4": 0.124532 } }, "true_positive": { "probabilities": { "false": 0.324773, "true": 0.675227 } }, "urgency": { "probabilities": { "0": 0.02944, "1": 0.10772, "2": 0.296958, "3": 0.565882 } } }, "security_incidents_000022": { "credential_compromise": { "probabilities": { "false": 0.600977, "true": 0.399023 } }, "disposition": { "probabilities": { "close_benign": 0.103928, "contain": 0.120523, "investigate": 0.553107, "monitor": 0.222442 } }, "severity": { "probabilities": { "0": 0.011251, "1": 0.075897, "2": 0.316598, "3": 0.491739, "4": 0.104515 } }, "true_positive": { "probabilities": { "false": 0.517668, "true": 0.482332 } }, "urgency": { "probabilities": { "0": 0.032671, "1": 0.192876, "2": 0.396939, "3": 0.377514 } } }, "security_incidents_000023": { "credential_compromise": { "probabilities": { "false": 0.821811, "true": 0.178189 } }, "disposition": { "probabilities": { "close_benign": 0.371928, "contain": 0.051777, "investigate": 0.339578, "monitor": 0.236716 } }, "severity": { "probabilities": { "0": 0.016364, "1": 0.073738, "2": 0.328403, "3": 0.491994, "4": 0.0895 } }, "true_positive": { "probabilities": { "false": 0.72123, "true": 0.27877 } }, "urgency": { "probabilities": { "0": 0.125395, "1": 0.303524, "2": 0.349789, "3": 0.221291 } } }, "security_incidents_000024": { "credential_compromise": { "probabilities": { "false": 0.506746, "true": 0.493254 } }, "disposition": { "probabilities": { "close_benign": 0.093889, "contain": 0.147944, "investigate": 0.568823, "monitor": 0.189344 } }, "severity": { "probabilities": { "0": 0.014569, "1": 0.054277, "2": 0.323834, "3": 0.525858, "4": 0.081462 } }, "true_positive": { "probabilities": { "false": 0.400202, "true": 0.599798 } }, "urgency": { "probabilities": { "0": 0.044142, "1": 0.155683, "2": 0.388093, "3": 0.412082 } } }, "security_incidents_000025": { "credential_compromise": { "probabilities": { "false": 0.402981, "true": 0.597019 } }, "disposition": { "probabilities": { "close_benign": 0.049044, "contain": 0.202168, "investigate": 0.604918, "monitor": 0.14387 } }, "severity": { "probabilities": { "0": 0.013816, "1": 0.112203, "2": 0.374905, "3": 0.40797, "4": 0.091106 } }, "true_positive": { "probabilities": { "false": 0.290609, "true": 0.709391 } }, "urgency": { "probabilities": { "0": 0.02334, "1": 0.143087, "2": 0.337556, "3": 0.496016 } } }, "security_incidents_000026": { "credential_compromise": { "probabilities": { "false": 0.539082, "true": 0.460918 } }, "disposition": { "probabilities": { "close_benign": 0.094906, "contain": 0.101662, "investigate": 0.612058, "monitor": 0.191374 } }, "severity": { "probabilities": { "0": 0.008138, "1": 0.051187, "2": 0.2504, "3": 0.525975, "4": 0.1643 } }, "true_positive": { "probabilities": { "false": 0.413047, "true": 0.586953 } }, "urgency": { "probabilities": { "0": 0.034614, "1": 0.138397, "2": 0.386525, "3": 0.440465 } } }, "security_incidents_000027": { "credential_compromise": { "probabilities": { "false": 0.361956, "true": 0.638044 } }, "disposition": { "probabilities": { "close_benign": 0.038844, "contain": 0.218142, "investigate": 0.619096, "monitor": 0.123918 } }, "severity": { "probabilities": { "0": 0.019776, "1": 0.127023, "2": 0.485736, "3": 0.322811, "4": 0.044654 } }, "true_positive": { "probabilities": { "false": 0.23218, "true": 0.76782 } }, "urgency": { "probabilities": { "0": 0.020264, "1": 0.134186, "2": 0.341762, "3": 0.503788 } } }, "security_incidents_000028": { "credential_compromise": { "probabilities": { "false": 0.251044, "true": 0.748956 } }, "disposition": { "probabilities": { "close_benign": 0.014316, "contain": 0.64059, "investigate": 0.301433, "monitor": 0.043661 } }, "severity": { "probabilities": { "0": 0.002419, "1": 0.010934, "2": 0.036422, "3": 0.167691, "4": 0.782534 } }, "true_positive": { "probabilities": { "false": 0.149833, "true": 0.850167 } }, "urgency": { "probabilities": { "0": 0.005883, "1": 0.0366, "2": 0.107022, "3": 0.850495 } } }, "security_incidents_000029": { "credential_compromise": { "probabilities": { "false": 0.342052, "true": 0.657948 } }, "disposition": { "probabilities": { "close_benign": 0.050495, "contain": 0.26434, "investigate": 0.564905, "monitor": 0.12026 } }, "severity": { "probabilities": { "0": 0.003718, "1": 0.017288, "2": 0.106778, "3": 0.616371, "4": 0.255845 } }, "true_positive": { "probabilities": { "false": 0.259975, "true": 0.740025 } }, "urgency": { "probabilities": { "0": 0.01698, "1": 0.06341, "2": 0.206668, "3": 0.712942 } } }, "security_incidents_000030": { "credential_compromise": { "probabilities": { "false": 0.221478, "true": 0.778522 } }, "disposition": { "probabilities": { "close_benign": 0.015545, "contain": 0.603798, "investigate": 0.337683, "monitor": 0.042974 } }, "severity": { "probabilities": { "0": 0.00206, "1": 0.012032, "2": 0.086655, "3": 0.398969, "4": 0.500284 } }, "true_positive": { "probabilities": { "false": 0.127131, "true": 0.872869 } }, "urgency": { "probabilities": { "0": 0.004305, "1": 0.02258, "2": 0.110207, "3": 0.862908 } } }, "security_incidents_000031": { "credential_compromise": { "probabilities": { "false": 0.253133, "true": 0.746867 } }, "disposition": { "probabilities": { "close_benign": 0.02033, "contain": 0.471276, "investigate": 0.446454, "monitor": 0.061941 } }, "severity": { "probabilities": { "0": 0.028315, "1": 0.108672, "2": 0.446428, "3": 0.352391, "4": 0.064194 } }, "true_positive": { "probabilities": { "false": 0.113898, "true": 0.886102 } }, "urgency": { "probabilities": { "0": 0.014617, "1": 0.064843, "2": 0.259103, "3": 0.661437 } } }, "security_incidents_000032": { "credential_compromise": { "probabilities": { "false": 0.774413, "true": 0.225587 } }, "disposition": { "probabilities": { "close_benign": 0.25946, "contain": 0.024965, "investigate": 0.358729, "monitor": 0.356846 } }, "severity": { "probabilities": { "0": 0.108583, "1": 0.336159, "2": 0.431228, "3": 0.110953, "4": 0.013077 } }, "true_positive": { "probabilities": { "false": 0.679403, "true": 0.320597 } }, "urgency": { "probabilities": { "0": 0.141287, "1": 0.477944, "2": 0.312969, "3": 0.0678 } } }, "security_incidents_000033": { "credential_compromise": { "probabilities": { "false": 0.569159, "true": 0.430841 } }, "disposition": { "probabilities": { "close_benign": 0.08398, "contain": 0.087582, "investigate": 0.581335, "monitor": 0.247103 } }, "severity": { "probabilities": { "0": 0.092686, "1": 0.362431, "2": 0.408084, "3": 0.120649, "4": 0.016149 } }, "true_positive": { "probabilities": { "false": 0.476062, "true": 0.523938 } }, "urgency": { "probabilities": { "0": 0.069294, "1": 0.293893, "2": 0.441969, "3": 0.194843 } } }, "security_incidents_000034": { "credential_compromise": { "probabilities": { "false": 0.302057, "true": 0.697943 } }, "disposition": { "probabilities": { "close_benign": 0.030603, "contain": 0.471197, "investigate": 0.419381, "monitor": 0.078819 } }, "severity": { "probabilities": { "0": 0.003081, "1": 0.01659, "2": 0.089223, "3": 0.475148, "4": 0.415957 } }, "true_positive": { "probabilities": { "false": 0.189583, "true": 0.810417 } }, "urgency": { "probabilities": { "0": 0.008154, "1": 0.047511, "2": 0.167537, "3": 0.776798 } } }, "security_incidents_000035": { "credential_compromise": { "probabilities": { "false": 0.268378, "true": 0.731622 } }, "disposition": { "probabilities": { "close_benign": 0.019401, "contain": 0.465704, "investigate": 0.450732, "monitor": 0.064163 } }, "severity": { "probabilities": { "0": 0.001421, "1": 0.007542, "2": 0.048702, "3": 0.33516, "4": 0.607175 } }, "true_positive": { "probabilities": { "false": 0.168369, "true": 0.831631 } }, "urgency": { "probabilities": { "0": 0.004827, "1": 0.031875, "2": 0.112062, "3": 0.851236 } } }, "security_incidents_000036": { "credential_compromise": { "probabilities": { "false": 0.377463, "true": 0.622537 } }, "disposition": { "probabilities": { "close_benign": 0.04451, "contain": 0.222805, "investigate": 0.598351, "monitor": 0.134334 } }, "severity": { "probabilities": { "0": 0.010903, "1": 0.066422, "2": 0.270333, "3": 0.528645, "4": 0.123697 } }, "true_positive": { "probabilities": { "false": 0.31724, "true": 0.68276 } }, "urgency": { "probabilities": { "0": 0.02402, "1": 0.116068, "2": 0.331023, "3": 0.528889 } } }, "security_incidents_000037": { "credential_compromise": { "probabilities": { "false": 0.435156, "true": 0.564844 } }, "disposition": { "probabilities": { "close_benign": 0.045757, "contain": 0.129462, "investigate": 0.644919, "monitor": 0.179862 } }, "severity": { "probabilities": { "0": 0.06677, "1": 0.302753, "2": 0.426414, "3": 0.179762, "4": 0.024302 } }, "true_positive": { "probabilities": { "false": 0.285517, "true": 0.714483 } }, "urgency": { "probabilities": { "0": 0.038268, "1": 0.23531, "2": 0.446509, "3": 0.279914 } } }, "security_incidents_000038": { "credential_compromise": { "probabilities": { "false": 0.397761, "true": 0.602239 } }, "disposition": { "probabilities": { "close_benign": 0.044871, "contain": 0.238102, "investigate": 0.600932, "monitor": 0.116095 } }, "severity": { "probabilities": { "0": 0.007678, "1": 0.048924, "2": 0.261647, "3": 0.524205, "4": 0.157546 } }, "true_positive": { "probabilities": { "false": 0.261078, "true": 0.738922 } }, "urgency": { "probabilities": { "0": 0.015375, "1": 0.075475, "2": 0.310397, "3": 0.598753 } } }, "security_incidents_000039": { "credential_compromise": { "probabilities": { "false": 0.225583, "true": 0.774417 } }, "disposition": { "probabilities": { "close_benign": 0.014826, "contain": 0.509795, "investigate": 0.405348, "monitor": 0.070031 } }, "severity": { "probabilities": { "0": 0.010635, "1": 0.053608, "2": 0.219461, "3": 0.492686, "4": 0.22361 } }, "true_positive": { "probabilities": { "false": 0.13688, "true": 0.86312 } }, "urgency": { "probabilities": { "0": 0.007712, "1": 0.049705, "2": 0.174123, "3": 0.76846 } } }, "security_incidents_000040": { "credential_compromise": { "probabilities": { "false": 0.505033, "true": 0.494967 } }, "disposition": { "probabilities": { "close_benign": 0.034605, "contain": 0.215762, "investigate": 0.623329, "monitor": 0.126304 } }, "severity": { "probabilities": { "0": 0.071306, "1": 0.363243, "2": 0.372712, "3": 0.164899, "4": 0.02784 } }, "true_positive": { "probabilities": { "false": 0.200321, "true": 0.799679 } }, "urgency": { "probabilities": { "0": 0.028659, "1": 0.231351, "2": 0.402451, "3": 0.337539 } } }, "security_incidents_000041": { "credential_compromise": { "probabilities": { "false": 0.731883, "true": 0.268117 } }, "disposition": { "probabilities": { "close_benign": 0.165565, "contain": 0.051588, "investigate": 0.529695, "monitor": 0.253152 } }, "severity": { "probabilities": { "0": 0.184705, "1": 0.438793, "2": 0.283204, "3": 0.080257, "4": 0.01304 } }, "true_positive": { "probabilities": { "false": 0.439508, "true": 0.560492 } }, "urgency": { "probabilities": { "0": 0.121215, "1": 0.419566, "2": 0.349225, "3": 0.109994 } } }, "security_incidents_000042": { "credential_compromise": { "probabilities": { "false": 0.551003, "true": 0.448997 } }, "disposition": { "probabilities": { "close_benign": 0.147658, "contain": 0.103224, "investigate": 0.528399, "monitor": 0.220719 } }, "severity": { "probabilities": { "0": 0.007916, "1": 0.039376, "2": 0.239469, "3": 0.597586, "4": 0.115653 } }, "true_positive": { "probabilities": { "false": 0.4746, "true": 0.5254 } }, "urgency": { "probabilities": { "0": 0.053113, "1": 0.193033, "2": 0.332643, "3": 0.421211 } } }, "security_incidents_000043": { "credential_compromise": { "probabilities": { "false": 0.188365, "true": 0.811635 } }, "disposition": { "probabilities": { "close_benign": 0.012939, "contain": 0.651738, "investigate": 0.301389, "monitor": 0.033934 } }, "severity": { "probabilities": { "0": 0.00173, "1": 0.010745, "2": 0.071243, "3": 0.32523, "4": 0.591052 } }, "true_positive": { "probabilities": { "false": 0.092951, "true": 0.907049 } }, "urgency": { "probabilities": { "0": 0.003372, "1": 0.018396, "2": 0.102356, "3": 0.875876 } } }, "security_incidents_000044": { "credential_compromise": { "probabilities": { "false": 0.614267, "true": 0.385733 } }, "disposition": { "probabilities": { "close_benign": 0.071628, "contain": 0.080514, "investigate": 0.552831, "monitor": 0.295027 } }, "severity": { "probabilities": { "0": 0.075629, "1": 0.301631, "2": 0.338508, "3": 0.251332, "4": 0.032899 } }, "true_positive": { "probabilities": { "false": 0.525897, "true": 0.474103 } }, "urgency": { "probabilities": { "0": 0.050633, "1": 0.377043, "2": 0.399812, "3": 0.172512 } } }, "security_incidents_000045": { "credential_compromise": { "probabilities": { "false": 0.436719, "true": 0.563281 } }, "disposition": { "probabilities": { "close_benign": 0.055608, "contain": 0.12187, "investigate": 0.624143, "monitor": 0.198379 } }, "severity": { "probabilities": { "0": 0.076734, "1": 0.316398, "2": 0.398293, "3": 0.184978, "4": 0.023597 } }, "true_positive": { "probabilities": { "false": 0.331853, "true": 0.668147 } }, "urgency": { "probabilities": { "0": 0.045993, "1": 0.256701, "2": 0.415613, "3": 0.281693 } } }, "security_incidents_000046": { "credential_compromise": { "probabilities": { "false": 0.358195, "true": 0.641805 } }, "disposition": { "probabilities": { "close_benign": 0.038633, "contain": 0.376518, "investigate": 0.478804, "monitor": 0.106045 } }, "severity": { "probabilities": { "0": 0.005321, "1": 0.02361, "2": 0.192881, "3": 0.548922, "4": 0.229266 } }, "true_positive": { "probabilities": { "false": 0.2608, "true": 0.7392 } }, "urgency": { "probabilities": { "0": 0.01248, "1": 0.070017, "2": 0.226398, "3": 0.691105 } } }, "security_incidents_000047": { "credential_compromise": { "probabilities": { "false": 0.607292, "true": 0.392708 } }, "disposition": { "probabilities": { "close_benign": 0.180833, "contain": 0.078387, "investigate": 0.475239, "monitor": 0.265541 } }, "severity": { "probabilities": { "0": 0.058378, "1": 0.187214, "2": 0.431571, "3": 0.28146, "4": 0.041378 } }, "true_positive": { "probabilities": { "false": 0.512863, "true": 0.487137 } }, "urgency": { "probabilities": { "0": 0.112698, "1": 0.313483, "2": 0.371503, "3": 0.202315 } } }, "security_incidents_000048": { "credential_compromise": { "probabilities": { "false": 0.726199, "true": 0.273801 } }, "disposition": { "probabilities": { "close_benign": 0.144759, "contain": 0.065214, "investigate": 0.553627, "monitor": 0.2364 } }, "severity": { "probabilities": { "0": 0.005878, "1": 0.032373, "2": 0.168468, "3": 0.660983, "4": 0.132298 } }, "true_positive": { "probabilities": { "false": 0.625013, "true": 0.374987 } }, "urgency": { "probabilities": { "0": 0.048656, "1": 0.232327, "2": 0.412305, "3": 0.306713 } } }, "security_incidents_000049": { "credential_compromise": { "probabilities": { "false": 0.60117, "true": 0.39883 } }, "disposition": { "probabilities": { "close_benign": 0.108396, "contain": 0.07101, "investigate": 0.549171, "monitor": 0.271423 } }, "severity": { "probabilities": { "0": 0.069553, "1": 0.296092, "2": 0.491143, "3": 0.127712, "4": 0.015501 } }, "true_positive": { "probabilities": { "false": 0.556432, "true": 0.443568 } }, "urgency": { "probabilities": { "0": 0.079709, "1": 0.374914, "2": 0.39349, "3": 0.151887 } } }, "security_incidents_000050": { "credential_compromise": { "probabilities": { "false": 0.24672, "true": 0.75328 } }, "disposition": { "probabilities": { "close_benign": 0.013092, "contain": 0.549812, "investigate": 0.382665, "monitor": 0.054431 } }, "severity": { "probabilities": { "0": 0.004303, "1": 0.025755, "2": 0.142083, "3": 0.413723, "4": 0.414136 } }, "true_positive": { "probabilities": { "false": 0.128226, "true": 0.871774 } }, "urgency": { "probabilities": { "0": 0.004448, "1": 0.030357, "2": 0.163228, "3": 0.801967 } } }, "security_incidents_000051": { "credential_compromise": { "probabilities": { "false": 0.742248, "true": 0.257752 } }, "disposition": { "probabilities": { "close_benign": 0.243482, "contain": 0.038578, "investigate": 0.397251, "monitor": 0.320689 } }, "severity": { "probabilities": { "0": 0.192612, "1": 0.373619, "2": 0.280101, "3": 0.135209, "4": 0.018459 } }, "true_positive": { "probabilities": { "false": 0.614362, "true": 0.385638 } }, "urgency": { "probabilities": { "0": 0.176937, "1": 0.443033, "2": 0.305923, "3": 0.074107 } } }, "security_incidents_000052": { "credential_compromise": { "probabilities": { "false": 0.477982, "true": 0.522018 } }, "disposition": { "probabilities": { "close_benign": 0.06482, "contain": 0.159824, "investigate": 0.61159, "monitor": 0.163766 } }, "severity": { "probabilities": { "0": 0.005745, "1": 0.031879, "2": 0.247302, "3": 0.609518, "4": 0.105556 } }, "true_positive": { "probabilities": { "false": 0.390724, "true": 0.609276 } }, "urgency": { "probabilities": { "0": 0.019746, "1": 0.108332, "2": 0.329283, "3": 0.542639 } } }, "security_incidents_000053": { "credential_compromise": { "probabilities": { "false": 0.560828, "true": 0.439172 } }, "disposition": { "probabilities": { "close_benign": 0.062633, "contain": 0.081693, "investigate": 0.644503, "monitor": 0.211171 } }, "severity": { "probabilities": { "0": 0.024294, "1": 0.128237, "2": 0.339116, "3": 0.469696, "4": 0.038657 } }, "true_positive": { "probabilities": { "false": 0.480593, "true": 0.519407 } }, "urgency": { "probabilities": { "0": 0.036953, "1": 0.242179, "2": 0.456468, "3": 0.2644 } } }, "security_incidents_000054": { "credential_compromise": { "probabilities": { "false": 0.410676, "true": 0.589324 } }, "disposition": { "probabilities": { "close_benign": 0.031102, "contain": 0.288574, "investigate": 0.58236, "monitor": 0.097964 } }, "severity": { "probabilities": { "0": 0.007621, "1": 0.047774, "2": 0.305407, "3": 0.508876, "4": 0.130322 } }, "true_positive": { "probabilities": { "false": 0.231989, "true": 0.768011 } }, "urgency": { "probabilities": { "0": 0.010606, "1": 0.067739, "2": 0.276523, "3": 0.645132 } } }, "security_incidents_000055": { "credential_compromise": { "probabilities": { "false": 0.6283, "true": 0.3717 } }, "disposition": { "probabilities": { "close_benign": 0.103471, "contain": 0.111932, "investigate": 0.568288, "monitor": 0.216309 } }, "severity": { "probabilities": { "0": 0.057032, "1": 0.176831, "2": 0.31125, "3": 0.384281, "4": 0.070606 } }, "true_positive": { "probabilities": { "false": 0.429007, "true": 0.570993 } }, "urgency": { "probabilities": { "0": 0.073133, "1": 0.239472, "2": 0.397796, "3": 0.2896 } } }, "security_incidents_000056": { "credential_compromise": { "probabilities": { "false": 0.825862, "true": 0.174138 } }, "disposition": { "probabilities": { "close_benign": 0.387092, "contain": 0.042529, "investigate": 0.305936, "monitor": 0.264443 } }, "severity": { "probabilities": { "0": 0.025267, "1": 0.092487, "2": 0.355784, "3": 0.446142, "4": 0.08032 } }, "true_positive": { "probabilities": { "false": 0.754524, "true": 0.245476 } }, "urgency": { "probabilities": { "0": 0.131481, "1": 0.367167, "2": 0.324124, "3": 0.177227 } } }, "security_incidents_000057": { "credential_compromise": { "probabilities": { "false": 0.630126, "true": 0.369874 } }, "disposition": { "probabilities": { "close_benign": 0.212748, "contain": 0.054187, "investigate": 0.460766, "monitor": 0.272299 } }, "severity": { "probabilities": { "0": 0.093725, "1": 0.25701, "2": 0.50295, "3": 0.130966, "4": 0.015349 } }, "true_positive": { "probabilities": { "false": 0.577499, "true": 0.422501 } }, "urgency": { "probabilities": { "0": 0.15717, "1": 0.32214, "2": 0.376265, "3": 0.144425 } } }, "security_incidents_000058": { "credential_compromise": { "probabilities": { "false": 0.3752, "true": 0.6248 } }, "disposition": { "probabilities": { "close_benign": 0.035455, "contain": 0.348409, "investigate": 0.512251, "monitor": 0.103885 } }, "severity": { "probabilities": { "0": 0.006922, "1": 0.041096, "2": 0.169748, "3": 0.352896, "4": 0.429338 } }, "true_positive": { "probabilities": { "false": 0.233664, "true": 0.766336 } }, "urgency": { "probabilities": { "0": 0.011764, "1": 0.077215, "2": 0.236969, "3": 0.674052 } } }, "security_incidents_000059": { "credential_compromise": { "probabilities": { "false": 0.497674, "true": 0.502326 } }, "disposition": { "probabilities": { "close_benign": 0.025616, "contain": 0.646283, "investigate": 0.294962, "monitor": 0.033139 } }, "severity": { "probabilities": { "0": 0.004062, "1": 0.016117, "2": 0.099272, "3": 0.309572, "4": 0.570977 } }, "true_positive": { "probabilities": { "false": 0.12635, "true": 0.87365 } }, "urgency": { "probabilities": { "0": 0.006612, "1": 0.034251, "2": 0.135635, "3": 0.823502 } } }, "security_incidents_000060": { "credential_compromise": { "probabilities": { "false": 0.913746, "true": 0.086254 } }, "disposition": { "probabilities": { "close_benign": 0.608125, "contain": 0.007683, "investigate": 0.12672, "monitor": 0.257472 } }, "severity": { "probabilities": { "0": 0.190266, "1": 0.399317, "2": 0.358192, "3": 0.047252, "4": 0.004973 } }, "true_positive": { "probabilities": { "false": 0.887891, "true": 0.112109 } }, "urgency": { "probabilities": { "0": 0.301109, "1": 0.530667, "2": 0.152024, "3": 0.0162 } } }, "security_incidents_000061": { "credential_compromise": { "probabilities": { "false": 0.284412, "true": 0.715588 } }, "disposition": { "probabilities": { "close_benign": 0.022261, "contain": 0.353421, "investigate": 0.537783, "monitor": 0.086535 } }, "severity": { "probabilities": { "0": 0.022203, "1": 0.112315, "2": 0.432023, "3": 0.33826, "4": 0.0952 } }, "true_positive": { "probabilities": { "false": 0.163618, "true": 0.836382 } }, "urgency": { "probabilities": { "0": 0.013831, "1": 0.112312, "2": 0.302363, "3": 0.571494 } } }, "security_incidents_000062": { "credential_compromise": { "probabilities": { "false": 0.759848, "true": 0.240152 } }, "disposition": { "probabilities": { "close_benign": 0.263618, "contain": 0.036135, "investigate": 0.410778, "monitor": 0.289469 } }, "severity": { "probabilities": { "0": 0.025154, "1": 0.094929, "2": 0.33094, "3": 0.46816, "4": 0.080816 } }, "true_positive": { "probabilities": { "false": 0.725466, "true": 0.274534 } }, "urgency": { "probabilities": { "0": 0.100778, "1": 0.360363, "2": 0.381664, "3": 0.157195 } } }, "security_incidents_000063": { "credential_compromise": { "probabilities": { "false": 0.353116, "true": 0.646884 } }, "disposition": { "probabilities": { "close_benign": 0.034925, "contain": 0.272383, "investigate": 0.575339, "monitor": 0.117353 } }, "severity": { "probabilities": { "0": 0.006832, "1": 0.038849, "2": 0.270639, "3": 0.561578, "4": 0.122102 } }, "true_positive": { "probabilities": { "false": 0.234634, "true": 0.765366 } }, "urgency": { "probabilities": { "0": 0.012662, "1": 0.071896, "2": 0.270947, "3": 0.644495 } } }, "security_incidents_000064": { "credential_compromise": { "probabilities": { "false": 0.420791, "true": 0.579209 } }, "disposition": { "probabilities": { "close_benign": 0.046416, "contain": 0.190384, "investigate": 0.62908, "monitor": 0.13412 } }, "severity": { "probabilities": { "0": 0.003272, "1": 0.021444, "2": 0.169113, "3": 0.628047, "4": 0.178124 } }, "true_positive": { "probabilities": { "false": 0.364494, "true": 0.635506 } }, "urgency": { "probabilities": { "0": 0.014451, "1": 0.092705, "2": 0.273739, "3": 0.619105 } } }, "security_incidents_000065": { "credential_compromise": { "probabilities": { "false": 0.371357, "true": 0.628643 } }, "disposition": { "probabilities": { "close_benign": 0.032825, "contain": 0.217805, "investigate": 0.594921, "monitor": 0.154449 } }, "severity": { "probabilities": { "0": 0.045062, "1": 0.221066, "2": 0.530282, "3": 0.168695, "4": 0.034895 } }, "true_positive": { "probabilities": { "false": 0.291256, "true": 0.708744 } }, "urgency": { "probabilities": { "0": 0.026173, "1": 0.204179, "2": 0.411195, "3": 0.358452 } } }, "security_incidents_000066": { "credential_compromise": { "probabilities": { "false": 0.38057, "true": 0.61943 } }, "disposition": { "probabilities": { "close_benign": 0.032776, "contain": 0.313577, "investigate": 0.544678, "monitor": 0.108969 } }, "severity": { "probabilities": { "0": 0.001362, "1": 0.010659, "2": 0.065476, "3": 0.472108, "4": 0.450395 } }, "true_positive": { "probabilities": { "false": 0.274227, "true": 0.725773 } }, "urgency": { "probabilities": { "0": 0.00756, "1": 0.046087, "2": 0.186195, "3": 0.760158 } } }, "security_incidents_000067": { "credential_compromise": { "probabilities": { "false": 0.700571, "true": 0.299429 } }, "disposition": { "probabilities": { "close_benign": 0.123072, "contain": 0.040405, "investigate": 0.519238, "monitor": 0.317285 } }, "severity": { "probabilities": { "0": 0.138543, "1": 0.47312, "2": 0.299162, "3": 0.076518, "4": 0.012657 } }, "true_positive": { "probabilities": { "false": 0.531761, "true": 0.468239 } }, "urgency": { "probabilities": { "0": 0.099262, "1": 0.483572, "2": 0.337491, "3": 0.079676 } } }, "security_incidents_000068": { "credential_compromise": { "probabilities": { "false": 0.777946, "true": 0.222054 } }, "disposition": { "probabilities": { "close_benign": 0.285117, "contain": 0.031998, "investigate": 0.37425, "monitor": 0.308635 } }, "severity": { "probabilities": { "0": 0.031079, "1": 0.114279, "2": 0.359468, "3": 0.424584, "4": 0.070589 } }, "true_positive": { "probabilities": { "false": 0.768892, "true": 0.231108 } }, "urgency": { "probabilities": { "0": 0.130363, "1": 0.356902, "2": 0.393999, "3": 0.118737 } } }, "security_incidents_000069": { "credential_compromise": { "probabilities": { "false": 0.337785, "true": 0.662215 } }, "disposition": { "probabilities": { "close_benign": 0.011773, "contain": 0.687954, "investigate": 0.268584, "monitor": 0.031689 } }, "severity": { "probabilities": { "0": 0.004668, "1": 0.022546, "2": 0.086608, "3": 0.271285, "4": 0.614893 } }, "true_positive": { "probabilities": { "false": 0.102426, "true": 0.897574 } }, "urgency": { "probabilities": { "0": 0.004324, "1": 0.032786, "2": 0.109236, "3": 0.853654 } } }, "security_incidents_000070": { "credential_compromise": { "probabilities": { "false": 0.613847, "true": 0.386153 } }, "disposition": { "probabilities": { "close_benign": 0.079648, "contain": 0.083846, "investigate": 0.576349, "monitor": 0.260157 } }, "severity": { "probabilities": { "0": 0.05944, "1": 0.225359, "2": 0.363308, "3": 0.316055, "4": 0.035838 } }, "true_positive": { "probabilities": { "false": 0.494916, "true": 0.505084 } }, "urgency": { "probabilities": { "0": 0.051543, "1": 0.286798, "2": 0.445399, "3": 0.216259 } } }, "security_incidents_000071": { "credential_compromise": { "probabilities": { "false": 0.289894, "true": 0.710106 } }, "disposition": { "probabilities": { "close_benign": 0.013376, "contain": 0.46019, "investigate": 0.473048, "monitor": 0.053387 } }, "severity": { "probabilities": { "0": 0.001599, "1": 0.009654, "2": 0.05357, "3": 0.260196, "4": 0.674981 } }, "true_positive": { "probabilities": { "false": 0.129454, "true": 0.870546 } }, "urgency": { "probabilities": { "0": 0.003297, "1": 0.026525, "2": 0.116163, "3": 0.854015 } } }, "security_incidents_000072": { "credential_compromise": { "probabilities": { "false": 0.798282, "true": 0.201718 } }, "disposition": { "probabilities": { "close_benign": 0.303672, "contain": 0.042755, "investigate": 0.389017, "monitor": 0.264556 } }, "severity": { "probabilities": { "0": 0.033561, "1": 0.107289, "2": 0.326152, "3": 0.317193, "4": 0.215805 } }, "true_positive": { "probabilities": { "false": 0.711772, "true": 0.288228 } }, "urgency": { "probabilities": { "0": 0.108059, "1": 0.336706, "2": 0.360372, "3": 0.194863 } } }, "security_incidents_000073": { "credential_compromise": { "probabilities": { "false": 0.535089, "true": 0.464911 } }, "disposition": { "probabilities": { "close_benign": 0.106708, "contain": 0.104334, "investigate": 0.573781, "monitor": 0.215177 } }, "severity": { "probabilities": { "0": 0.028735, "1": 0.12303, "2": 0.444754, "3": 0.341228, "4": 0.062253 } }, "true_positive": { "probabilities": { "false": 0.414436, "true": 0.585564 } }, "urgency": { "probabilities": { "0": 0.050769, "1": 0.274316, "2": 0.392293, "3": 0.282623 } } }, "security_incidents_000074": { "credential_compromise": { "probabilities": { "false": 0.749978, "true": 0.250022 } }, "disposition": { "probabilities": { "close_benign": 0.18293, "contain": 0.093157, "investigate": 0.494629, "monitor": 0.229284 } }, "severity": { "probabilities": { "0": 0.014559, "1": 0.06187, "2": 0.308507, "3": 0.522097, "4": 0.092967 } }, "true_positive": { "probabilities": { "false": 0.576945, "true": 0.423055 } }, "urgency": { "probabilities": { "0": 0.059715, "1": 0.246155, "2": 0.380577, "3": 0.313553 } } }, "security_incidents_000075": { "credential_compromise": { "probabilities": { "false": 0.407426, "true": 0.592574 } }, "disposition": { "probabilities": { "close_benign": 0.046925, "contain": 0.1137, "investigate": 0.671432, "monitor": 0.167943 } }, "severity": { "probabilities": { "0": 0.084095, "1": 0.337308, "2": 0.384063, "3": 0.166304, "4": 0.028229 } }, "true_positive": { "probabilities": { "false": 0.272925, "true": 0.727075 } }, "urgency": { "probabilities": { "0": 0.045869, "1": 0.219416, "2": 0.41433, "3": 0.320385 } } }, "security_incidents_000076": { "credential_compromise": { "probabilities": { "false": 0.235114, "true": 0.764886 } }, "disposition": { "probabilities": { "close_benign": 0.014059, "contain": 0.538281, "investigate": 0.40115, "monitor": 0.04651 } }, "severity": { "probabilities": { "0": 0.001158, "1": 0.006149, "2": 0.044438, "3": 0.340539, "4": 0.607716 } }, "true_positive": { "probabilities": { "false": 0.135989, "true": 0.864011 } }, "urgency": { "probabilities": { "0": 0.003089, "1": 0.021238, "2": 0.092908, "3": 0.882765 } } }, "security_incidents_000077": { "credential_compromise": { "probabilities": { "false": 0.699154, "true": 0.300846 } }, "disposition": { "probabilities": { "close_benign": 0.182333, "contain": 0.030237, "investigate": 0.467714, "monitor": 0.319716 } }, "severity": { "probabilities": { "0": 0.149537, "1": 0.436622, "2": 0.311385, "3": 0.092536, "4": 0.009921 } }, "true_positive": { "probabilities": { "false": 0.603203, "true": 0.396797 } }, "urgency": { "probabilities": { "0": 0.147452, "1": 0.43478, "2": 0.336608, "3": 0.081159 } } }, "security_incidents_000078": { "credential_compromise": { "probabilities": { "false": 0.237553, "true": 0.762447 } }, "disposition": { "probabilities": { "close_benign": 0.013708, "contain": 0.548098, "investigate": 0.380745, "monitor": 0.057449 } }, "severity": { "probabilities": { "0": 0.001318, "1": 0.006137, "2": 0.036905, "3": 0.268128, "4": 0.687512 } }, "true_positive": { "probabilities": { "false": 0.177564, "true": 0.822436 } }, "urgency": { "probabilities": { "0": 0.004209, "1": 0.026635, "2": 0.116094, "3": 0.853062 } } }, "security_incidents_000079": { "credential_compromise": { "probabilities": { "false": 0.724217, "true": 0.275783 } }, "disposition": { "probabilities": { "close_benign": 0.365622, "contain": 0.044247, "investigate": 0.319841, "monitor": 0.27029 } }, "severity": { "probabilities": { "0": 0.143669, "1": 0.252078, "2": 0.237551, "3": 0.32515, "4": 0.041552 } }, "true_positive": { "probabilities": { "false": 0.607563, "true": 0.392437 } }, "urgency": { "probabilities": { "0": 0.227445, "1": 0.396023, "2": 0.254428, "3": 0.122104 } } }, "security_incidents_000080": { "credential_compromise": { "probabilities": { "false": 0.391728, "true": 0.608272 } }, "disposition": { "probabilities": { "close_benign": 0.043911, "contain": 0.264349, "investigate": 0.578225, "monitor": 0.113515 } }, "severity": { "probabilities": { "0": 0.00635, "1": 0.033023, "2": 0.145988, "3": 0.337606, "4": 0.477033 } }, "true_positive": { "probabilities": { "false": 0.292413, "true": 0.707587 } }, "urgency": { "probabilities": { "0": 0.012968, "1": 0.074753, "2": 0.215674, "3": 0.696605 } } }, "security_incidents_000081": { "credential_compromise": { "probabilities": { "false": 0.703884, "true": 0.296116 } }, "disposition": { "probabilities": { "close_benign": 0.157516, "contain": 0.100605, "investigate": 0.507499, "monitor": 0.23438 } }, "severity": { "probabilities": { "0": 0.006789, "1": 0.026275, "2": 0.13807, "3": 0.450271, "4": 0.378595 } }, "true_positive": { "probabilities": { "false": 0.577569, "true": 0.422431 } }, "urgency": { "probabilities": { "0": 0.042677, "1": 0.18975, "2": 0.35852, "3": 0.409052 } } }, "security_incidents_000082": { "credential_compromise": { "probabilities": { "false": 0.528993, "true": 0.471007 } }, "disposition": { "probabilities": { "close_benign": 0.092419, "contain": 0.099449, "investigate": 0.635844, "monitor": 0.172288 } }, "severity": { "probabilities": { "0": 0.008216, "1": 0.040724, "2": 0.272661, "3": 0.6117, "4": 0.066699 } }, "true_positive": { "probabilities": { "false": 0.463333, "true": 0.536667 } }, "urgency": { "probabilities": { "0": 0.046675, "1": 0.160817, "2": 0.413263, "3": 0.379246 } } }, "security_incidents_000083": { "credential_compromise": { "probabilities": { "false": 0.385489, "true": 0.614511 } }, "disposition": { "probabilities": { "close_benign": 0.055105, "contain": 0.214603, "investigate": 0.5905, "monitor": 0.139792 } }, "severity": { "probabilities": { "0": 0.044037, "1": 0.217041, "2": 0.331766, "3": 0.299152, "4": 0.108005 } }, "true_positive": { "probabilities": { "false": 0.230251, "true": 0.769749 } }, "urgency": { "probabilities": { "0": 0.04396, "1": 0.182474, "2": 0.367317, "3": 0.406249 } } }, "security_incidents_000084": { "credential_compromise": { "probabilities": { "false": 0.619671, "true": 0.380329 } }, "disposition": { "probabilities": { "close_benign": 0.16543, "contain": 0.061388, "investigate": 0.553416, "monitor": 0.219766 } }, "severity": { "probabilities": { "0": 0.049362, "1": 0.167874, "2": 0.520083, "3": 0.233112, "4": 0.029569 } }, "true_positive": { "probabilities": { "false": 0.439158, "true": 0.560842 } }, "urgency": { "probabilities": { "0": 0.093148, "1": 0.27039, "2": 0.415961, "3": 0.220501 } } }, "security_incidents_000085": { "credential_compromise": { "probabilities": { "false": 0.289751, "true": 0.710249 } }, "disposition": { "probabilities": { "close_benign": 0.042395, "contain": 0.321975, "investigate": 0.528508, "monitor": 0.107122 } }, "severity": { "probabilities": { "0": 0.003742, "1": 0.021668, "2": 0.137564, "3": 0.640814, "4": 0.196212 } }, "true_positive": { "probabilities": { "false": 0.244608, "true": 0.755392 } }, "urgency": { "probabilities": { "0": 0.014406, "1": 0.070241, "2": 0.190671, "3": 0.724682 } } }, "security_incidents_000086": { "credential_compromise": { "probabilities": { "false": 0.621864, "true": 0.378136 } }, "disposition": { "probabilities": { "close_benign": 0.194667, "contain": 0.047121, "investigate": 0.470634, "monitor": 0.287578 } }, "severity": { "probabilities": { "0": 0.059455, "1": 0.21773, "2": 0.514484, "3": 0.184087, "4": 0.024244 } }, "true_positive": { "probabilities": { "false": 0.531777, "true": 0.468223 } }, "urgency": { "probabilities": { "0": 0.113796, "1": 0.37169, "2": 0.358351, "3": 0.156163 } } }, "security_incidents_000087": { "credential_compromise": { "probabilities": { "false": 0.326026, "true": 0.673974 } }, "disposition": { "probabilities": { "close_benign": 0.027114, "contain": 0.324724, "investigate": 0.552609, "monitor": 0.095553 } }, "severity": { "probabilities": { "0": 0.005609, "1": 0.040727, "2": 0.192394, "3": 0.481699, "4": 0.27957 } }, "true_positive": { "probabilities": { "false": 0.205329, "true": 0.794671 } }, "urgency": { "probabilities": { "0": 0.009081, "1": 0.055063, "2": 0.239354, "3": 0.696502 } } }, "security_incidents_000088": { "credential_compromise": { "probabilities": { "false": 0.49777, "true": 0.50223 } }, "disposition": { "probabilities": { "close_benign": 0.068027, "contain": 0.185391, "investigate": 0.578874, "monitor": 0.167708 } }, "severity": { "probabilities": { "0": 0.02038, "1": 0.101991, "2": 0.435995, "3": 0.325089, "4": 0.116545 } }, "true_positive": { "probabilities": { "false": 0.340787, "true": 0.659213 } }, "urgency": { "probabilities": { "0": 0.032847, "1": 0.1966, "2": 0.391556, "3": 0.378996 } } }, "security_incidents_000089": { "credential_compromise": { "probabilities": { "false": 0.544062, "true": 0.455938 } }, "disposition": { "probabilities": { "close_benign": 0.108487, "contain": 0.15413, "investigate": 0.548109, "monitor": 0.189274 } }, "severity": { "probabilities": { "0": 0.008231, "1": 0.046833, "2": 0.225532, "3": 0.544789, "4": 0.174615 } }, "true_positive": { "probabilities": { "false": 0.466036, "true": 0.533964 } }, "urgency": { "probabilities": { "0": 0.033984, "1": 0.140711, "2": 0.332263, "3": 0.493042 } } }, "security_incidents_000090": { "credential_compromise": { "probabilities": { "false": 0.288903, "true": 0.711097 } }, "disposition": { "probabilities": { "close_benign": 0.022794, "contain": 0.280716, "investigate": 0.583278, "monitor": 0.113212 } }, "severity": { "probabilities": { "0": 0.04644, "1": 0.193576, "2": 0.402932, "3": 0.282869, "4": 0.074183 } }, "true_positive": { "probabilities": { "false": 0.199692, "true": 0.800308 } }, "urgency": { "probabilities": { "0": 0.019273, "1": 0.128849, "2": 0.363232, "3": 0.488646 } } }, "security_incidents_000091": { "credential_compromise": { "probabilities": { "false": 0.501835, "true": 0.498165 } }, "disposition": { "probabilities": { "close_benign": 0.05877, "contain": 0.256822, "investigate": 0.541404, "monitor": 0.143004 } }, "severity": { "probabilities": { "0": 0.010837, "1": 0.060089, "2": 0.21667, "3": 0.372234, "4": 0.34017 } }, "true_positive": { "probabilities": { "false": 0.356837, "true": 0.643163 } }, "urgency": { "probabilities": { "0": 0.019729, "1": 0.129538, "2": 0.282954, "3": 0.567779 } } }, "security_incidents_000092": { "credential_compromise": { "probabilities": { "false": 0.751335, "true": 0.248665 } }, "disposition": { "probabilities": { "close_benign": 0.186753, "contain": 0.071453, "investigate": 0.513745, "monitor": 0.228049 } }, "severity": { "probabilities": { "0": 0.019442, "1": 0.067413, "2": 0.328349, "3": 0.493879, "4": 0.090916 } }, "true_positive": { "probabilities": { "false": 0.568304, "true": 0.431696 } }, "urgency": { "probabilities": { "0": 0.066626, "1": 0.276041, "2": 0.395634, "3": 0.261699 } } }, "security_incidents_000093": { "credential_compromise": { "probabilities": { "false": 0.663198, "true": 0.336802 } }, "disposition": { "probabilities": { "close_benign": 0.111414, "contain": 0.050415, "investigate": 0.469528, "monitor": 0.368643 } }, "severity": { "probabilities": { "0": 0.151597, "1": 0.526727, "2": 0.260992, "3": 0.054618, "4": 0.006066 } }, "true_positive": { "probabilities": { "false": 0.571373, "true": 0.428627 } }, "urgency": { "probabilities": { "0": 0.090977, "1": 0.485736, "2": 0.339863, "3": 0.083423 } } }, "security_incidents_000094": { "credential_compromise": { "probabilities": { "false": 0.358212, "true": 0.641788 } }, "disposition": { "probabilities": { "close_benign": 0.033814, "contain": 0.413389, "investigate": 0.462258, "monitor": 0.090539 } }, "severity": { "probabilities": { "0": 0.007837, "1": 0.032419, "2": 0.231493, "3": 0.568073, "4": 0.160178 } }, "true_positive": { "probabilities": { "false": 0.212099, "true": 0.787901 } }, "urgency": { "probabilities": { "0": 0.011321, "1": 0.057844, "2": 0.215609, "3": 0.715226 } } }, "security_incidents_000095": { "credential_compromise": { "probabilities": { "false": 0.643361, "true": 0.356639 } }, "disposition": { "probabilities": { "close_benign": 0.197985, "contain": 0.092327, "investigate": 0.468536, "monitor": 0.241151 } }, "severity": { "probabilities": { "0": 0.01368, "1": 0.073988, "2": 0.302108, "3": 0.543417, "4": 0.066807 } }, "true_positive": { "probabilities": { "false": 0.593377, "true": 0.406623 } }, "urgency": { "probabilities": { "0": 0.086194, "1": 0.228719, "2": 0.360916, "3": 0.324171 } } }, "security_incidents_000096": { "credential_compromise": { "probabilities": { "false": 0.398021, "true": 0.601979 } }, "disposition": { "probabilities": { "close_benign": 0.040944, "contain": 0.270817, "investigate": 0.580333, "monitor": 0.107906 } }, "severity": { "probabilities": { "0": 0.006791, "1": 0.035568, "2": 0.282232, "3": 0.564975, "4": 0.110434 } }, "true_positive": { "probabilities": { "false": 0.263857, "true": 0.736143 } }, "urgency": { "probabilities": { "0": 0.013434, "1": 0.073443, "2": 0.28402, "3": 0.629103 } } }, "security_incidents_000097": { "credential_compromise": { "probabilities": { "false": 0.52628, "true": 0.47372 } }, "disposition": { "probabilities": { "close_benign": 0.084443, "contain": 0.103033, "investigate": 0.59635, "monitor": 0.216174 } }, "severity": { "probabilities": { "0": 0.030398, "1": 0.134052, "2": 0.411725, "3": 0.381256, "4": 0.04257 } }, "true_positive": { "probabilities": { "false": 0.480578, "true": 0.519422 } }, "urgency": { "probabilities": { "0": 0.05183, "1": 0.232338, "2": 0.443039, "3": 0.272793 } } }, "security_incidents_000098": { "credential_compromise": { "probabilities": { "false": 0.477835, "true": 0.522165 } }, "disposition": { "probabilities": { "close_benign": 0.043266, "contain": 0.130256, "investigate": 0.655443, "monitor": 0.171035 } }, "severity": { "probabilities": { "0": 0.048411, "1": 0.201387, "2": 0.33005, "3": 0.364117, "4": 0.056035 } }, "true_positive": { "probabilities": { "false": 0.307641, "true": 0.692359 } }, "urgency": { "probabilities": { "0": 0.032571, "1": 0.195183, "2": 0.426585, "3": 0.345661 } } }, "security_incidents_000099": { "credential_compromise": { "probabilities": { "false": 0.45742, "true": 0.54258 } }, "disposition": { "probabilities": { "close_benign": 0.030715, "contain": 0.303535, "investigate": 0.549148, "monitor": 0.116602 } }, "severity": { "probabilities": { "0": 0.040817, "1": 0.145859, "2": 0.293955, "3": 0.433068, "4": 0.0863 } }, "true_positive": { "probabilities": { "false": 0.230541, "true": 0.769459 } }, "urgency": { "probabilities": { "0": 0.022546, "1": 0.14872, "2": 0.339664, "3": 0.48907 } } }, "security_incidents_000016": { "credential_compromise": { "probabilities": { "false": 0.383761, "true": 0.616239 } }, "disposition": { "probabilities": { "close_benign": 0.038039, "contain": 0.318088, "investigate": 0.524567, "monitor": 0.119306 } }, "severity": { "probabilities": { "0": 0.011756, "1": 0.077673, "2": 0.190187, "3": 0.327555, "4": 0.392829 } }, "true_positive": { "probabilities": { "false": 0.30491, "true": 0.69509 } }, "urgency": { "probabilities": { "0": 0.018308, "1": 0.096853, "2": 0.234487, "3": 0.650352 } } }, "security_incidents_000017": { "credential_compromise": { "probabilities": { "false": 0.349384, "true": 0.650616 } }, "disposition": { "probabilities": { "close_benign": 0.049742, "contain": 0.219702, "investigate": 0.588941, "monitor": 0.141615 } }, "severity": { "probabilities": { "0": 0.08158, "1": 0.342659, "2": 0.359053, "3": 0.182065, "4": 0.034644 } }, "true_positive": { "probabilities": { "false": 0.244342, "true": 0.755658 } }, "urgency": { "probabilities": { "0": 0.046312, "1": 0.196175, "2": 0.357741, "3": 0.399772 } } }, "security_incidents_000018": { "credential_compromise": { "probabilities": { "false": 0.599645, "true": 0.400355 } }, "disposition": { "probabilities": { "close_benign": 0.164409, "contain": 0.087866, "investigate": 0.528343, "monitor": 0.219382 } }, "severity": { "probabilities": { "0": 0.02676, "1": 0.115604, "2": 0.428063, "3": 0.372483, "4": 0.05709 } }, "true_positive": { "probabilities": { "false": 0.492516, "true": 0.507484 } }, "urgency": { "probabilities": { "0": 0.073893, "1": 0.239963, "2": 0.37675, "3": 0.309394 } } }, "security_incidents_000019": { "credential_compromise": { "probabilities": { "false": 0.516263, "true": 0.483737 } }, "disposition": { "probabilities": { "close_benign": 0.106349, "contain": 0.115818, "investigate": 0.609405, "monitor": 0.168428 } }, "severity": { "probabilities": { "0": 0.006181, "1": 0.031042, "2": 0.215649, "3": 0.619202, "4": 0.127926 } }, "true_positive": { "probabilities": { "false": 0.412457, "true": 0.587543 } }, "urgency": { "probabilities": { "0": 0.03767, "1": 0.111338, "2": 0.345454, "3": 0.505538 } } } }, "per_case": [ { "id": "agent_trace_observability_000000", "workflow": "agent_trace_observability", "decisions": [ { "type": "choice", "correct": true, "kl": 0.24640572990077236, "brier": 0.096721315572, "confidence": 0.542038, "qid": "action" }, { "type": "noul", "correct": true, "kl": 0.13752105564190958, "brier": 0.08935032644999998, "confidence": 0.854698, "qid": "needs_review" }, { "type": "choice", "correct": true, "kl": 0.0598539427464774, "brier": 0.033551878147999994, "confidence": 0.740354, "qid": "outcome" }, { "type": "score", "correct": true, "kl": 0.03903126733088237, "brier": 0.007700724694999996, "pred_score": 0.9444790000000001, "gold_score": 0.79, "confidence": 0.564276, "qid": "risk" }, { "type": "score", "correct": true, "kl": 0.014645936386224936, "brier": 0.0025006416619999952, "pred_score": 0.636031, "gold_score": 0.556667, "confidence": 0.517619, "qid": "urgency" } ] }, { "id": "agent_trace_observability_000001", "workflow": "agent_trace_observability", "decisions": [ { "type": "choice", "correct": true, "kl": 0.0545403919364464, "brier": 0.027319713293999993, "confidence": 0.468914, "qid": "action" }, { "type": "noul", "correct": true, "kl": 0.03606488992840469, "brier": 0.031897317887999994, "confidence": 0.623712, "qid": "needs_review" }, { "type": "choice", "correct": false, "kl": 0.17499618688963248, "brier": 0.125423124977, "confidence": 0.398393, "qid": "outcome" }, { "type": "score", "correct": false, "kl": 0.030876171077383135, "brier": 0.009370848317097466, "pred_score": 0.7116942883057118, "gold_score": 0.556667, "confidence": 0.4623395376604624, "qid": "risk" }, { "type": "score", "correct": true, "kl": 0.011699625335901936, "brier": 0.0058425237270000015, "pred_score": 0.836005, "gold_score": 0.72, "confidence": 0.453194, "qid": "urgency" } ] }, { "id": "agent_trace_observability_000002", "workflow": "agent_trace_observability", "decisions": [ { "type": "choice", "correct": false, "kl": 0.2753612224078584, "brier": 0.117664113943, "confidence": 0.456646, "qid": "action" }, { "type": "noul", "correct": false, "kl": 0.057713514354433226, "brier": 0.057255926407999994, "confidence": 0.575865, "qid": "needs_review" }, { "type": "choice", "correct": false, "kl": 0.4914580881091649, "brier": 0.20858294036006728, "confidence": 0.4886495113504887, "qid": "outcome" }, { "type": "score", "correct": true, "kl": 0.018851451200413594, "brier": 0.002848742178408506, "pred_score": 0.743898256101744, "gold_score": 0.7, "confidence": 0.4881835118164882, "qid": "risk" }, { "type": "score", "correct": true, "kl": 0.08990967573050707, "brier": 0.03416195312999999, "pred_score": 1.076325, "gold_score": 1.316667, "confidence": 0.519258, "qid": "urgency" } ] }, { "id": "agent_trace_observability_000003", "workflow": "agent_trace_observability", "decisions": [ { "type": "choice", "correct": false, "kl": 0.07572052239922153, "brier": 0.046355760814999994, "confidence": 0.464356, "qid": "action" }, { "type": "noul", "correct": true, "kl": 0.018198762187727327, "brier": 0.016284710450000014, "confidence": 0.690235, "qid": "needs_review" }, { "type": "choice", "correct": false, "kl": 0.029720580556538603, "brier": 0.011437078898000004, "confidence": 0.433132, "qid": "outcome" }, { "type": "score", "correct": true, "kl": 0.01907735216858513, "brier": 0.011959098009000003, "pred_score": 0.733953, "gold_score": 0.74, "confidence": 0.476017, "qid": "risk" }, { "type": "score", "correct": true, "kl": 0.1079766668528988, "brier": 0.08205463978251613, "pred_score": 0.7260892739107261, "gold_score": 0.983333, "confidence": 0.4506005493994506, "qid": "urgency" } ] }, { "id": "agent_trace_observability_000004", "workflow": "agent_trace_observability", "decisions": [ { "type": "choice", "correct": false, "kl": 0.016548807883096192, "brier": 0.011987038925999994, "confidence": 0.442228, "qid": "action" }, { "type": "noul", "correct": true, "kl": 9.790860794948964e-07, "brier": 8.217620000000069e-07, "confidence": 0.700641, "qid": "needs_review" }, { "type": "choice", "correct": false, "kl": 0.09545311601084834, "brier": 0.06603976219236346, "confidence": 0.4742984742984743, "qid": "outcome" }, { "type": "score", "correct": false, "kl": 0.1941093624009374, "brier": 0.13561856062300004, "pred_score": 0.688204, "gold_score": 0.3, "confidence": 0.478549, "qid": "risk" }, { "type": "score", "correct": false, "kl": 0.28311042473572684, "brier": 0.19170907184484196, "pred_score": 0.8095991904008097, "gold_score": 0.3, "confidence": 0.4625285374714626, "qid": "urgency" } ] }, { "id": "agent_trace_observability_000005", "workflow": "agent_trace_observability", "decisions": [ { "type": "choice", "correct": false, "kl": 0.20128017557545763, "brier": 0.11620953358600002, "confidence": 0.454533, "qid": "action" }, { "type": "noul", "correct": true, "kl": 0.05501538533016136, "brier": 0.039964168327999997, "confidence": 0.708642, "qid": "needs_review" }, { "type": "choice", "correct": true, "kl": 0.20593856753608955, "brier": 0.14853545316200004, "confidence": 0.496028, "qid": "outcome" }, { "type": "score", "correct": true, "kl": 0.01885430941122411, "brier": 0.010774910346999999, "pred_score": 0.9926810000000001, "gold_score": 0.966667, "confidence": 0.6283150000000001, "qid": "risk" }, { "type": "score", "correct": false, "kl": 0.06182634773385971, "brier": 0.045193326936000006, "pred_score": 0.9638979999999999, "gold_score": 0.916667, "confidence": 0.488293, "qid": "urgency" } ] }, { "id": "agent_trace_observability_000006", "workflow": "agent_trace_observability", "decisions": [ { "type": "choice", "correct": true, "kl": 0.01846856863377614, "brier": 0.009643077074000013, "confidence": 0.6080760000000001, "qid": "action" }, { "type": "noul", "correct": true, "kl": 0.0005023081890454043, "brier": 0.000191140352, "confidence": 0.890224, "qid": "needs_review" }, { "type": "choice", "correct": false, "kl": 0.3798998185260079, "brier": 0.24632278092300006, "confidence": 0.733581, "qid": "outcome" }, { "type": "score", "correct": false, "kl": 0.2106986824379951, "brier": 0.169858043085, "pred_score": 2.631416, "gold_score": 2.343333, "confidence": 0.709759, "qid": "risk" }, { "type": "score", "correct": true, "kl": 0.03470742883910925, "brier": 0.026062539537000006, "pred_score": 2.407213, "gold_score": 2.233333, "confidence": 0.562922, "qid": "urgency" } ] }, { "id": "agent_trace_observability_000007", "workflow": "agent_trace_observability", "decisions": [ { "type": "choice", "correct": true, "kl": 0.0044860415161915475, "brier": 0.000570553358000001, "confidence": 0.446915, "qid": "action" }, { "type": "noul", "correct": false, "kl": 0.003767837810763154, "brier": 0.0037639958480000005, "confidence": 0.526715, "qid": "needs_review" }, { "type": "choice", "correct": true, "kl": 0.027205369485136667, "brier": 0.013396734023999999, "confidence": 0.503236, "qid": "outcome" }, { "type": "score", "correct": true, "kl": 0.02833803502692775, "brier": 0.013153354458000002, "pred_score": 0.830213, "gold_score": 0.8, "confidence": 0.588063, "qid": "risk" }, { "type": "score", "correct": true, "kl": 0.14955165910834334, "brier": 0.04645385317799999, "pred_score": 0.975281, "gold_score": 1.383333, "confidence": 0.522031, "qid": "urgency" } ] }, { "id": "agent_trace_observability_000008", "workflow": "agent_trace_observability", "decisions": [ { "type": "choice", "correct": true, "kl": 0.05989305148932819, "brier": 0.01991170411300001, "confidence": 0.633811, "qid": "action" }, { "type": "noul", "correct": true, "kl": 0.14678361737600829, "brier": 0.06729826593799998, "confidence": 0.91677, "qid": "needs_review" }, { "type": "choice", "correct": true, "kl": 0.07716773431122231, "brier": 0.04973434300600001, "confidence": 0.653803, "qid": "outcome" }, { "type": "score", "correct": true, "kl": 0.016845018153570437, "brier": 0.010682597843999988, "pred_score": 2.544298, "gold_score": 2.44, "confidence": 0.631799, "qid": "risk" }, { "type": "score", "correct": true, "kl": 0.02270173894793262, "brier": 0.004047540828999995, "pred_score": 2.328617, "gold_score": 2.2, "confidence": 0.486268, "qid": "urgency" } ] }, { "id": "agent_trace_observability_000009", "workflow": "agent_trace_observability", "decisions": [ { "type": "choice", "correct": true, "kl": 0.008600683802822716, "brier": 0.0038571617899999977, "confidence": 0.610156, "qid": "action" }, { "type": "noul", "correct": true, "kl": 0.00044142153623420895, "brier": 0.0001598471999999991, "confidence": 0.902273, "qid": "needs_review" }, { "type": "choice", "correct": true, "kl": 0.014024351779035538, "brier": 0.005167396217999996, "confidence": 0.716865, "qid": "outcome" }, { "type": "score", "correct": true, "kl": 0.01592506969909545, "brier": 0.001216836798, "pred_score": 2.5965650000000005, "gold_score": 2.583333, "confidence": 0.6689, "qid": "risk" }, { "type": "score", "correct": true, "kl": 0.03079631983432722, "brier": 0.021344523129999998, "pred_score": 2.3974930000000003, "gold_score": 2.216667, "confidence": 0.537524, "qid": "urgency" } ] }, { "id": "agent_trace_observability_000010", "workflow": "agent_trace_observability", "decisions": [ { "type": "choice", "correct": false, "kl": 0.6048237038891591, "brier": 0.41458236845599994, "confidence": 0.465053, "qid": "action" }, { "type": "noul", "correct": true, "kl": 0.10199531309278706, "brier": 0.068145459488, "confidence": 0.715412, "qid": "needs_review" }, { "type": "choice", "correct": false, "kl": 0.6958597278569404, "brier": 0.5125495542454657, "confidence": 0.49412949412949414, "qid": "outcome" }, { "type": "score", "correct": true, "kl": 0.07157679023774277, "brier": 0.02299650936600001, "pred_score": 0.910925, "gold_score": 0.683667, "confidence": 0.625851, "qid": "risk" }, { "type": "score", "correct": true, "kl": 0.11578221444480806, "brier": 0.029390756146, "pred_score": 0.904758, "gold_score": 0.633667, "confidence": 0.509718, "qid": "urgency" } ] }, { "id": "agent_trace_observability_000011", "workflow": "agent_trace_observability", "decisions": [ { "type": "choice", "correct": true, "kl": 0.10279583146598385, "brier": 0.07205270478200003, "confidence": 0.566016, "qid": "action" }, { "type": "noul", "correct": true, "kl": 0.026250119191249234, "brier": 0.010098762962, "confidence": 0.865608, "qid": "needs_review" }, { "type": "choice", "correct": true, "kl": 0.084268509480904, "brier": 0.053359787012999954, "confidence": 0.679417, "qid": "outcome" }, { "type": "score", "correct": true, "kl": 0.00717364254001388, "brier": 0.004657785781000006, "pred_score": 0.465687, "gold_score": 0.416667, "confidence": 0.596356, "qid": "risk" }, { "type": "score", "correct": true, "kl": 0.004358091051642034, "brier": 0.0035193959260000065, "pred_score": 0.411867, "gold_score": 0.456667, "confidence": 0.653147, "qid": "urgency" } ] }, { "id": "agent_trace_observability_000012", "workflow": "agent_trace_observability", "decisions": [ { "type": "choice", "correct": true, "kl": 0.02752276661977929, "brier": 0.014487045456000006, "confidence": 0.40711, "qid": "action" }, { "type": "noul", "correct": true, "kl": 2.8996330137540452e-05, "brier": 2.7483698000000234e-05, "confidence": 0.61296, "qid": "needs_review" }, { "type": "choice", "correct": true, "kl": 0.014214164315753716, "brier": 0.006390342916036692, "confidence": 0.4658784658784659, "qid": "outcome" }, { "type": "score", "correct": true, "kl": 0.06469321101846792, "brier": 0.02707685241475398, "pred_score": 1.3670673670673672, "gold_score": 1.5, "confidence": 0.41821341821341823, "qid": "risk" }, { "type": "score", "correct": false, "kl": 0.14840473881898147, "brier": 0.07542698593499998, "pred_score": 0.889091, "gold_score": 1.183333, "confidence": 0.430868, "qid": "urgency" } ] }, { "id": "agent_trace_observability_000013", "workflow": "agent_trace_observability", "decisions": [ { "type": "choice", "correct": false, "kl": 0.024726696173343686, "brier": 0.012635571384000009, "confidence": 0.453909, "qid": "action" }, { "type": "noul", "correct": true, "kl": 0.009518805175189896, "brier": 0.007563024071999992, "confidence": 0.705173, "qid": "needs_review" }, { "type": "choice", "correct": true, "kl": 0.03164643924470244, "brier": 0.022253762878999994, "confidence": 0.522593, "qid": "outcome" }, { "type": "score", "correct": false, "kl": 0.06409022077176085, "brier": 0.05064827968199998, "pred_score": 0.763218, "gold_score": 0.553333, "confidence": 0.559132, "qid": "risk" }, { "type": "score", "correct": false, "kl": 0.11140124781954841, "brier": 0.09149437283699996, "pred_score": 0.843182, "gold_score": 0.656667, "confidence": 0.520527, "qid": "urgency" } ] }, { "id": "agent_trace_observability_000014", "workflow": "agent_trace_observability", "decisions": [ { "type": "choice", "correct": false, "kl": 0.31942390498792655, "brier": 0.15607366089699998, "confidence": 0.438409, "qid": "action" }, { "type": "noul", "correct": true, "kl": 0.1596466558851154, "brier": 0.142164983538, "confidence": 0.55672, "qid": "needs_review" }, { "type": "choice", "correct": false, "kl": 0.2820229706292154, "brier": 0.178065383094, "confidence": 0.448692, "qid": "outcome" }, { "type": "score", "correct": true, "kl": 0.02423669122838256, "brier": 0.005507666168000005, "pred_score": 1.108649, "gold_score": 1.02, "confidence": 0.549958, "qid": "risk" }, { "type": "score", "correct": true, "kl": 0.07088010825505268, "brier": 0.028574627904999993, "pred_score": 1.130903, "gold_score": 0.853333, "confidence": 0.501181, "qid": "urgency" } ] }, { "id": "agent_trace_observability_000015", "workflow": "agent_trace_observability", "decisions": [ { "type": "choice", "correct": true, "kl": 0.13111569256930214, "brier": 0.029256893139, "confidence": 0.559466, "qid": "action" }, { "type": "noul", "correct": true, "kl": 0.0017119560330270851, "brier": 0.0006752812500000012, "confidence": 0.895042, "qid": "needs_review" }, { "type": "choice", "correct": true, "kl": 0.1241095159985306, "brier": 0.041356604432999995, "confidence": 0.727727, "qid": "outcome" }, { "type": "score", "correct": true, "kl": 0.052249552028463084, "brier": 0.027383703199000017, "pred_score": 2.600075, "gold_score": 2.376667, "confidence": 0.672574, "qid": "risk" }, { "type": "score", "correct": false, "kl": 0.047241293254755845, "brier": 0.037175732107, "pred_score": 2.4032400000000003, "gold_score": 2.2, "confidence": 0.553169, "qid": "urgency" } ] }, { "id": "agent_trace_observability_000016", "workflow": "agent_trace_observability", "decisions": [ { "type": "choice", "correct": true, "kl": 0.02783734393602863, "brier": 0.0028647317580000036, "confidence": 0.706204, "qid": "action" }, { "type": "noul", "correct": true, "kl": 0.07085588469782812, "brier": 0.025810828807999994, "confidence": 0.930269, "qid": "needs_review" }, { "type": "choice", "correct": true, "kl": 0.017486880180512546, "brier": 0.011416539033999995, "confidence": 0.562162, "qid": "outcome" }, { "type": "score", "correct": true, "kl": 0.014652809035895234, "brier": 0.011661794422000007, "pred_score": 2.436533, "gold_score": 2.533333, "confidence": 0.550624, "qid": "risk" }, { "type": "score", "correct": true, "kl": 0.0462207310582925, "brier": 0.03531343247500001, "pred_score": 2.242833, "gold_score": 2.35, "confidence": 0.421757, "qid": "urgency" } ] }, { "id": "agent_trace_observability_000017", "workflow": "agent_trace_observability", "decisions": [ { "type": "choice", "correct": false, "kl": 0.04552875901889533, "brier": 0.02888614301426278, "confidence": 0.3909043909043909, "qid": "action" }, { "type": "noul", "correct": false, "kl": 0.10022482618560738, "brier": 0.09821489280799997, "confidence": 0.528269, "qid": "needs_review" }, { "type": "choice", "correct": true, "kl": 0.011707787264643929, "brier": 0.005008251851, "confidence": 0.464462, "qid": "outcome" }, { "type": "score", "correct": true, "kl": 0.13095575826416703, "brier": 0.055284789253298355, "pred_score": 1.5160345160345159, "gold_score": 1.883333, "confidence": 0.4661944661944662, "qid": "risk" }, { "type": "score", "correct": false, "kl": 0.44502250989913916, "brier": 0.162752604238, "pred_score": 1.24378, "gold_score": 1.966667, "confidence": 0.482315, "qid": "urgency" } ] }, { "id": "agent_trace_observability_000018", "workflow": "agent_trace_observability", "decisions": [ { "type": "choice", "correct": true, "kl": 0.10568335655279458, "brier": 0.074481652405, "confidence": 0.521669, "qid": "action" }, { "type": "noul", "correct": true, "kl": 0.03711910101589255, "brier": 0.021064191752, "confidence": 0.790707, "qid": "needs_review" }, { "type": "choice", "correct": true, "kl": 0.2060871092367369, "brier": 0.16346660357799997, "confidence": 0.487775, "qid": "outcome" }, { "type": "score", "correct": true, "kl": 0.025195959707820292, "brier": 0.003989257676999997, "pred_score": 0.328044, "gold_score": 0.303333, "confidence": 0.728532, "qid": "risk" }, { "type": "score", "correct": true, "kl": 0.005856381061297896, "brier": 0.0023937188140000024, "pred_score": 0.4376460000000001, "gold_score": 0.503333, "confidence": 0.6342220000000001, "qid": "urgency" } ] }, { "id": "agent_trace_observability_000019", "workflow": "agent_trace_observability", "decisions": [ { "type": "choice", "correct": false, "kl": 0.19640397005689947, "brier": 0.09270151287675303, "confidence": 0.4144044144044144, "qid": "action" }, { "type": "noul", "correct": true, "kl": 0.06189749082557307, "brier": 0.05015704608800001, "confidence": 0.658305, "qid": "needs_review" }, { "type": "choice", "correct": true, "kl": 0.05553041724083729, "brier": 0.03999637036437242, "confidence": 0.4533024533024533, "qid": "outcome" }, { "type": "score", "correct": false, "kl": 0.2803613929155568, "brier": 0.1331115163691991, "pred_score": 1.1938418061581937, "gold_score": 0.65, "confidence": 0.3628266371733628, "qid": "risk" }, { "type": "score", "correct": false, "kl": 0.16735182532704934, "brier": 0.10819325264900004, "pred_score": 0.803712, "gold_score": 0.4, "confidence": 0.431794, "qid": "urgency" } ] }, { "id": "agent_trace_observability_000020", "workflow": "agent_trace_observability", "decisions": [ { "type": "choice", "correct": true, "kl": 0.057160784810141016, "brier": 0.021448860990000006, "confidence": 0.405871, "qid": "action" }, { "type": "noul", "correct": true, "kl": 0.027507624608131928, "brier": 0.025185310178000006, "confidence": 0.60445, "qid": "needs_review" }, { "type": "choice", "correct": true, "kl": 0.10684124401519746, "brier": 0.06557234913799999, "confidence": 0.659458, "qid": "outcome" }, { "type": "score", "correct": true, "kl": 0.03585303152003282, "brier": 0.004740580005999999, "pred_score": 1.619732, "gold_score": 1.516667, "confidence": 0.551751, "qid": "risk" }, { "type": "score", "correct": true, "kl": 0.05916580764052692, "brier": 0.017392953632000007, "pred_score": 1.014695, "gold_score": 0.77, "confidence": 0.486784, "qid": "urgency" } ] }, { "id": "agent_trace_observability_000021", "workflow": "agent_trace_observability", "decisions": [ { "type": "choice", "correct": true, "kl": 0.0051232347431344775, "brier": 0.000836133645000001, "confidence": 0.663876, "qid": "action" }, { "type": "noul", "correct": true, "kl": 0.00436879919748288, "brier": 0.0016469208319999967, "confidence": 0.884637, "qid": "needs_review" }, { "type": "choice", "correct": true, "kl": 0.05824311788766655, "brier": 0.029826086506, "confidence": 0.642988, "qid": "outcome" }, { "type": "score", "correct": true, "kl": 0.1080484719556287, "brier": 0.06433341379300002, "pred_score": 0.340672, "gold_score": 0.12, "confidence": 0.709032, "qid": "risk" }, { "type": "score", "correct": true, "kl": 0.1693824074368151, "brier": 0.09167760490600001, "pred_score": 0.355287, "gold_score": 0.73, "confidence": 0.690412, "qid": "urgency" } ] }, { "id": "agent_trace_observability_000022", "workflow": "agent_trace_observability", "decisions": [ { "type": "choice", "correct": true, "kl": 0.01631589549952874, "brier": 0.004482753721534336, "confidence": 0.4227924227924228, "qid": "action" }, { "type": "noul", "correct": true, "kl": 0.010793489513702764, "brier": 0.008627462081999989, "confidence": 0.700988, "qid": "needs_review" }, { "type": "choice", "correct": true, "kl": 0.045367194716759554, "brier": 0.010058766122000008, "confidence": 0.513423, "qid": "outcome" }, { "type": "score", "correct": true, "kl": 0.025396139639022083, "brier": 0.016159435337999985, "pred_score": 0.29098, "gold_score": 0.413333, "confidence": 0.759103, "qid": "risk" }, { "type": "score", "correct": true, "kl": 0.0042794117343462335, "brier": 0.0012762036609999994, "pred_score": 0.598723, "gold_score": 0.623333, "confidence": 0.526593, "qid": "urgency" } ] }, { "id": "agent_trace_observability_000023", "workflow": "agent_trace_observability", "decisions": [ { "type": "choice", "correct": false, "kl": 0.14860084555038786, "brier": 0.10158280427000002, "confidence": 0.38514, "qid": "action" }, { "type": "noul", "correct": true, "kl": 0.06293893677117705, "brier": 0.03797602641800002, "confidence": 0.762203, "qid": "needs_review" }, { "type": "choice", "correct": true, "kl": 0.2297688726304301, "brier": 0.151809289429, "confidence": 0.393571, "qid": "outcome" }, { "type": "score", "correct": true, "kl": 0.03232410961931991, "brier": 0.023747117449999983, "pred_score": 0.470349, "gold_score": 0.633333, "confidence": 0.606219, "qid": "risk" }, { "type": "score", "correct": false, "kl": 0.10779552373350684, "brier": 0.07739223167899999, "pred_score": 0.547818, "gold_score": 0.87, "confidence": 0.576915, "qid": "urgency" } ] }, { "id": "agent_trace_observability_000024", "workflow": "agent_trace_observability", "decisions": [ { "type": "choice", "correct": true, "kl": 0.023754534219141383, "brier": 0.010665370403000007, "confidence": 0.544923, "qid": "action" }, { "type": "noul", "correct": true, "kl": 0.03437508831204386, "brier": 0.03230932840200001, "confidence": 0.660434, "qid": "needs_review" }, { "type": "choice", "correct": false, "kl": 0.24519658428123683, "brier": 0.15024927261867765, "confidence": 0.4738505261494738, "qid": "outcome" }, { "type": "score", "correct": true, "kl": 0.023346797629550338, "brier": 0.010618209189000003, "pred_score": 1.706856, "gold_score": 1.566667, "confidence": 0.544017, "qid": "risk" }, { "type": "score", "correct": true, "kl": 0.018639973087916963, "brier": 0.012448938525000003, "pred_score": 1.4539849999999999, "gold_score": 1.333333, "confidence": 0.412676, "qid": "urgency" } ] }, { "id": "agent_trace_observability_000025", "workflow": "agent_trace_observability", "decisions": [ { "type": "choice", "correct": true, "kl": 0.026604499708393257, "brier": 0.0062959381840000045, "confidence": 0.648258, "qid": "action" }, { "type": "noul", "correct": true, "kl": 0.0684887430727182, "brier": 0.029000507778, "confidence": 0.91375, "qid": "needs_review" }, { "type": "choice", "correct": true, "kl": 0.09172407419119849, "brier": 0.052877421786000016, "confidence": 0.6801590000000001, "qid": "outcome" }, { "type": "score", "correct": true, "kl": 0.018882243804734522, "brier": 0.012969658030000015, "pred_score": 2.5277350000000003, "gold_score": 2.4, "confidence": 0.625615, "qid": "risk" }, { "type": "score", "correct": true, "kl": 0.004771780056793757, "brier": 0.0028574171740000016, "pred_score": 2.330679, "gold_score": 2.336667, "confidence": 0.488189, "qid": "urgency" } ] }, { "id": "agent_trace_observability_000026", "workflow": "agent_trace_observability", "decisions": [ { "type": "choice", "correct": true, "kl": 0.18831862562523025, "brier": 0.048854078131999996, "confidence": 0.659926, "qid": "action" }, { "type": "noul", "correct": true, "kl": 0.08211471625033256, "brier": 0.02861262576199999, "confidence": 0.936276, "qid": "needs_review" }, { "type": "choice", "correct": true, "kl": 0.04239874244486669, "brier": 0.024700752807000004, "confidence": 0.617723, "qid": "outcome" }, { "type": "score", "correct": false, "kl": 0.11897699094176559, "brier": 0.08158211853200002, "pred_score": 2.5732150000000003, "gold_score": 2.246667, "confidence": 0.648996, "qid": "risk" }, { "type": "score", "correct": true, "kl": 0.04638672460057981, "brier": 0.021178080802000007, "pred_score": 2.4207650000000003, "gold_score": 2.193333, "confidence": 0.541552, "qid": "urgency" } ] }, { "id": "agent_trace_observability_000027", "workflow": "agent_trace_observability", "decisions": [ { "type": "choice", "correct": true, "kl": 0.08663070821674364, "brier": 0.05549880012600001, "confidence": 0.445401, "qid": "action" }, { "type": "noul", "correct": true, "kl": 0.015021633024156766, "brier": 0.013050170568000004, "confidence": 0.652555, "qid": "needs_review" }, { "type": "choice", "correct": false, "kl": 0.2543889088576462, "brier": 0.177169930694, "confidence": 0.457145, "qid": "outcome" }, { "type": "score", "correct": true, "kl": 0.15211872427719925, "brier": 0.066294444954, "pred_score": 0.949048, "gold_score": 0.583333, "confidence": 0.582191, "qid": "risk" }, { "type": "score", "correct": true, "kl": 0.03981607849159526, "brier": 0.004854973977999999, "pred_score": 1.04899, "gold_score": 0.966667, "confidence": 0.509414, "qid": "urgency" } ] }, { "id": "agent_trace_observability_000028", "workflow": "agent_trace_observability", "decisions": [ { "type": "choice", "correct": true, "kl": 0.012993763332004658, "brier": 0.003902937943000003, "confidence": 0.584495, "qid": "action" }, { "type": "noul", "correct": true, "kl": 5.3433103088789975e-05, "brier": 2.3474951999999597e-05, "confidence": 0.873241, "qid": "needs_review" }, { "type": "choice", "correct": true, "kl": 0.021718573851481543, "brier": 0.003639448465999998, "confidence": 0.71041, "qid": "outcome" }, { "type": "score", "correct": true, "kl": 0.019249317575083396, "brier": 0.003813797047000001, "pred_score": 0.254473, "gold_score": 0.336667, "confidence": 0.78171, "qid": "risk" }, { "type": "score", "correct": true, "kl": 0.03863694846775148, "brier": 0.017526608693000004, "pred_score": 0.30248, "gold_score": 0.453333, "confidence": 0.740616, "qid": "urgency" } ] }, { "id": "agent_trace_observability_000029", "workflow": "agent_trace_observability", "decisions": [ { "type": "choice", "correct": false, "kl": 0.008030329724138504, "brier": 0.005608464269999997, "confidence": 0.40557, "qid": "action" }, { "type": "noul", "correct": true, "kl": 0.004043767652633222, "brier": 0.004035433122000003, "confidence": 0.505081, "qid": "needs_review" }, { "type": "choice", "correct": true, "kl": 0.09287247878453304, "brier": 0.06705738016200001, "confidence": 0.455018, "qid": "outcome" }, { "type": "score", "correct": true, "kl": 0.04073959001328286, "brier": 0.018643192293, "pred_score": 0.8844510000000001, "gold_score": 0.766667, "confidence": 0.571134, "qid": "risk" }, { "type": "score", "correct": true, "kl": 0.00786504156678159, "brier": 0.0009037785500000001, "pred_score": 1.018439, "gold_score": 1.05, "confidence": 0.496038, "qid": "urgency" } ] }, { "id": "agent_trace_observability_000030", "workflow": "agent_trace_observability", "decisions": [ { "type": "choice", "correct": true, "kl": 0.016612056826173276, "brier": 0.004363114983, "confidence": 0.538796, "qid": "action" }, { "type": "noul", "correct": true, "kl": 0.0174712426533625, "brier": 0.015015058632, "confidence": 0.656687, "qid": "needs_review" }, { "type": "choice", "correct": true, "kl": 0.019819411014038692, "brier": 0.005626617222000002, "confidence": 0.518435, "qid": "outcome" }, { "type": "score", "correct": false, "kl": 0.032456677647082516, "brier": 0.017996818660428633, "pred_score": 1.5451584548415451, "gold_score": 1.383333, "confidence": 0.4932025067974932, "qid": "risk" }, { "type": "score", "correct": true, "kl": 0.022805612805885493, "brier": 0.011064174264999994, "pred_score": 1.3269499999999999, "gold_score": 1.483333, "confidence": 0.502867, "qid": "urgency" } ] }, { "id": "agent_trace_observability_000031", "workflow": "agent_trace_observability", "decisions": [ { "type": "choice", "correct": true, "kl": 0.00530694966029711, "brier": 0.001592238321999998, "confidence": 0.485783, "qid": "action" }, { "type": "noul", "correct": true, "kl": 0.0034091272353274082, "brier": 0.0032001600020000057, "confidence": 0.609999, "qid": "needs_review" }, { "type": "choice", "correct": true, "kl": 0.02638159747224579, "brier": 0.02299190693100001, "confidence": 0.586802, "qid": "outcome" }, { "type": "score", "correct": true, "kl": 0.04987152727103744, "brier": 0.03944227522200001, "pred_score": 0.834591, "gold_score": 0.916667, "confidence": 0.478456, "qid": "risk" }, { "type": "score", "correct": true, "kl": 0.03904266388829741, "brier": 0.015530377421999999, "pred_score": 1.169648, "gold_score": 1.3, "confidence": 0.48818, "qid": "urgency" } ] }, { "id": "agent_trace_observability_000032", "workflow": "agent_trace_observability", "decisions": [ { "type": "choice", "correct": true, "kl": 0.08420710261883446, "brier": 0.037487520066999985, "confidence": 0.698226, "qid": "action" }, { "type": "noul", "correct": true, "kl": 0.020437418564283008, "brier": 0.010422503441999992, "confidence": 0.872189, "qid": "needs_review" }, { "type": "choice", "correct": true, "kl": 0.027885614008700464, "brier": 0.018268559988041735, "confidence": 0.4966375033624967, "qid": "outcome" }, { "type": "score", "correct": true, "kl": 0.0034670829035068837, "brier": 0.0021907607730000007, "pred_score": 2.231983, "gold_score": 2.2, "confidence": 0.467332, "qid": "risk" }, { "type": "score", "correct": true, "kl": 0.01702994399494398, "brier": 0.009120969464000003, "pred_score": 1.90041, "gold_score": 2.05, "confidence": 0.393555, "qid": "urgency" } ] }, { "id": "agent_trace_observability_000033", "workflow": "agent_trace_observability", "decisions": [ { "type": "choice", "correct": true, "kl": 0.11658471369474618, "brier": 0.041447476694000014, "confidence": 0.667316, "qid": "action" }, { "type": "noul", "correct": true, "kl": 0.007156616877408668, "brier": 0.0023105042419999952, "confidence": 0.899344, "qid": "needs_review" }, { "type": "choice", "correct": true, "kl": 0.09771679718076053, "brier": 0.027803377422, "confidence": 0.657728, "qid": "outcome" }, { "type": "score", "correct": true, "kl": 0.07040170934271461, "brier": 0.03862754606499999, "pred_score": 2.46215, "gold_score": 2.7, "confidence": 0.571809, "qid": "risk" }, { "type": "score", "correct": true, "kl": 0.07926889189956868, "brier": 0.036109062857001495, "pred_score": 2.2320757679242322, "gold_score": 2.516667, "confidence": 0.4376005623994377, "qid": "urgency" } ] }, { "id": "agent_trace_observability_000034", "workflow": "agent_trace_observability", "decisions": [ { "type": "choice", "correct": false, "kl": 0.18192908563761404, "brier": 0.09433002926199999, "confidence": 0.421199, "qid": "action" }, { "type": "noul", "correct": false, "kl": 0.030811643043255427, "brier": 0.0305588642, "confidence": 0.506943, "qid": "needs_review" }, { "type": "choice", "correct": true, "kl": 0.018856262691735584, "brier": 0.013650286928999986, "confidence": 0.505376, "qid": "outcome" }, { "type": "score", "correct": true, "kl": 0.03512892678449244, "brier": 0.024610283810000022, "pred_score": 1.644487, "gold_score": 1.693333, "confidence": 0.514782, "qid": "risk" }, { "type": "score", "correct": true, "kl": 0.014853393525427843, "brier": 0.007336707611277577, "pred_score": 1.2685507314492688, "gold_score": 1.216667, "confidence": 0.4356765643234357, "qid": "urgency" } ] }, { "id": "agent_trace_observability_000035", "workflow": "agent_trace_observability", "decisions": [ { "type": "choice", "correct": false, "kl": 0.1411715817324361, "brier": 0.092373020152, "confidence": 0.476027, "qid": "action" }, { "type": "noul", "correct": true, "kl": 0.020449739859705426, "brier": 0.016927999999999988, "confidence": 0.674667, "qid": "needs_review" }, { "type": "choice", "correct": false, "kl": 0.14755875878401586, "brier": 0.10188256573800002, "confidence": 0.515572, "qid": "outcome" }, { "type": "score", "correct": true, "kl": 0.004659997822110867, "brier": 0.0006239105339999999, "pred_score": 0.5885090000000001, "gold_score": 0.603333, "confidence": 0.5036080000000001, "qid": "risk" }, { "type": "score", "correct": false, "kl": 0.01976555562734196, "brier": 0.015320717946765589, "pred_score": 0.7536867536867536, "gold_score": 0.69, "confidence": 0.4589294589294589, "qid": "urgency" } ] }, { "id": "agent_trace_observability_000036", "workflow": "agent_trace_observability", "decisions": [ { "type": "choice", "correct": false, "kl": 0.036113047048275476, "brier": 0.019395947268000004, "confidence": 0.41495, "qid": "action" }, { "type": "noul", "correct": false, "kl": 0.06945734135793194, "brier": 0.067863700872, "confidence": 0.500873, "qid": "needs_review" }, { "type": "choice", "correct": true, "kl": 0.0269986966992974, "brier": 0.007335431851999994, "confidence": 0.510369, "qid": "outcome" }, { "type": "score", "correct": true, "kl": 0.03574103370337338, "brier": 0.022894860549000005, "pred_score": 0.543715, "gold_score": 0.4, "confidence": 0.55118, "qid": "risk" }, { "type": "score", "correct": true, "kl": 0.01761972765885345, "brier": 0.009089950933999991, "pred_score": 1.015128, "gold_score": 0.966667, "confidence": 0.520496, "qid": "urgency" } ] }, { "id": "agent_trace_observability_000037", "workflow": "agent_trace_observability", "decisions": [ { "type": "choice", "correct": true, "kl": 0.1011643528341793, "brier": 0.025278782822000002, "confidence": 0.647838, "qid": "action" }, { "type": "noul", "correct": true, "kl": 0.08396817970013407, "brier": 0.040041670050000006, "confidence": 0.901495, "qid": "needs_review" }, { "type": "choice", "correct": true, "kl": 0.03650168993752451, "brier": 0.022876416421999984, "confidence": 0.681653, "qid": "outcome" }, { "type": "score", "correct": true, "kl": 0.03232153382572582, "brier": 0.022236792327000002, "pred_score": 2.536989, "gold_score": 2.37, "confidence": 0.630791, "qid": "risk" }, { "type": "score", "correct": true, "kl": 0.0012489987033815876, "brier": 0.0002813394220000008, "pred_score": 2.342418, "gold_score": 2.316667, "confidence": 0.504046, "qid": "urgency" } ] }, { "id": "agent_trace_observability_000038", "workflow": "agent_trace_observability", "decisions": [ { "type": "choice", "correct": true, "kl": 0.022568487185619022, "brier": 0.005357126522000001, "confidence": 0.513856, "qid": "action" }, { "type": "noul", "correct": true, "kl": 0.0010287967820598182, "brier": 0.0009707299219999996, "confidence": 0.611302, "qid": "needs_review" }, { "type": "choice", "correct": false, "kl": 0.03265728242052057, "brier": 0.02073551515099999, "confidence": 0.420964, "qid": "outcome" }, { "type": "score", "correct": false, "kl": 0.06422656464741969, "brier": 0.05650019448999999, "pred_score": 0.690117, "gold_score": 0.51, "confidence": 0.463663, "qid": "risk" }, { "type": "score", "correct": false, "kl": 0.02142420063105117, "brier": 0.009869236757352684, "pred_score": 0.7501022498977501, "gold_score": 0.82, "confidence": 0.4428155571844428, "qid": "urgency" } ] }, { "id": "agent_trace_observability_000039", "workflow": "agent_trace_observability", "decisions": [ { "type": "choice", "correct": true, "kl": 0.06623072413907348, "brier": 0.03449501511899999, "confidence": 0.6153460000000001, "qid": "action" }, { "type": "noul", "correct": true, "kl": 0.0020253470808188484, "brier": 0.0008134964480000027, "confidence": 0.879832, "qid": "needs_review" }, { "type": "choice", "correct": true, "kl": 0.1508670319025885, "brier": 0.07798643925300003, "confidence": 0.756884, "qid": "outcome" }, { "type": "score", "correct": true, "kl": 0.07748626561343716, "brier": 0.05699638317400002, "pred_score": 2.620382, "gold_score": 2.466667, "confidence": 0.683262, "qid": "risk" }, { "type": "score", "correct": false, "kl": 0.16011762735721674, "brier": 0.108012369697, "pred_score": 2.46076, "gold_score": 2.066667, "confidence": 0.602722, "qid": "urgency" } ] }, { "id": "agent_trace_observability_000040", "workflow": "agent_trace_observability", "decisions": [ { "type": "choice", "correct": true, "kl": 0.22776121314366987, "brier": 0.058503959354000004, "confidence": 0.452757, "qid": "action" }, { "type": "noul", "correct": true, "kl": 0.09162983576363817, "brier": 0.084252662018, "confidence": 0.56142, "qid": "needs_review" }, { "type": "choice", "correct": true, "kl": 0.11708059335654246, "brier": 0.08734341128400001, "confidence": 0.460988, "qid": "outcome" }, { "type": "score", "correct": false, "kl": 0.3023177122768399, "brier": 0.23065086237800003, "pred_score": 0.887888, "gold_score": 0.423333, "confidence": 0.576047, "qid": "risk" }, { "type": "score", "correct": true, "kl": 0.029944851368023377, "brier": 0.008445375225000001, "pred_score": 1.1195490000000001, "gold_score": 1.266667, "confidence": 0.528483, "qid": "urgency" } ] }, { "id": "agent_trace_observability_000041", "workflow": "agent_trace_observability", "decisions": [ { "type": "choice", "correct": false, "kl": 0.23214221996699352, "brier": 0.13868861192, "confidence": 0.498135, "qid": "action" }, { "type": "noul", "correct": true, "kl": 0.03265520395674283, "brier": 0.024759355392000006, "confidence": 0.705403, "qid": "needs_review" }, { "type": "choice", "correct": false, "kl": 0.17996109225797652, "brier": 0.12803085414635432, "confidence": 0.4956535043464957, "qid": "outcome" }, { "type": "score", "correct": true, "kl": 0.01984718292289372, "brier": 0.007347823789999991, "pred_score": 0.44310200000000005, "gold_score": 0.333333, "confidence": 0.6285810000000001, "qid": "risk" }, { "type": "score", "correct": true, "kl": 0.049695530242179445, "brier": 0.016521085933999997, "pred_score": 0.622682, "gold_score": 0.433333, "confidence": 0.493455, "qid": "urgency" } ] }, { "id": "agent_trace_observability_000042", "workflow": "agent_trace_observability", "decisions": [ { "type": "choice", "correct": false, "kl": 0.12960152099459693, "brier": 0.06413124964199998, "confidence": 0.423737, "qid": "action" }, { "type": "noul", "correct": true, "kl": 0.015499646658234889, "brier": 0.015365044999999992, "confidence": 0.51235, "qid": "needs_review" }, { "type": "choice", "correct": false, "kl": 0.22023811864571857, "brier": 0.12541032289399998, "confidence": 0.529669, "qid": "outcome" }, { "type": "score", "correct": true, "kl": 0.08317909755463769, "brier": 0.05909402036200002, "pred_score": 0.661612, "gold_score": 0.396667, "confidence": 0.454597, "qid": "risk" }, { "type": "score", "correct": false, "kl": 0.31998242017142686, "brier": 0.188944205614, "pred_score": 1.0342099999999999, "gold_score": 0.473333, "confidence": 0.52228, "qid": "urgency" } ] }, { "id": "agent_trace_observability_000043", "workflow": "agent_trace_observability", "decisions": [ { "type": "choice", "correct": true, "kl": 0.025884467739938723, "brier": 0.006762279566, "confidence": 0.554257, "qid": "action" }, { "type": "noul", "correct": true, "kl": 0.001091141376212381, "brier": 0.0008909264720000035, "confidence": 0.721106, "qid": "needs_review" }, { "type": "choice", "correct": true, "kl": 0.02158015116458427, "brier": 0.007725836677999997, "confidence": 0.433047, "qid": "outcome" }, { "type": "score", "correct": true, "kl": 0.07698921000583274, "brier": 0.015738815482000002, "pred_score": 1.969458, "gold_score": 1.7, "confidence": 0.603604, "qid": "risk" }, { "type": "score", "correct": true, "kl": 0.0005963993928722773, "brier": 0.00034300651299999955, "pred_score": 1.755915, "gold_score": 1.75, "confidence": 0.419874, "qid": "urgency" } ] }, { "id": "agent_trace_observability_000044", "workflow": "agent_trace_observability", "decisions": [ { "type": "choice", "correct": true, "kl": 0.019395796487572066, "brier": 0.013629627721999986, "confidence": 0.527572, "qid": "action" }, { "type": "noul", "correct": true, "kl": 0.014580143006400967, "brier": 0.013000136258, "confidence": 0.636044, "qid": "needs_review" }, { "type": "choice", "correct": true, "kl": 0.041385250611240806, "brier": 0.013472763818999998, "confidence": 0.476258, "qid": "outcome" }, { "type": "score", "correct": true, "kl": 0.01993629731081292, "brier": 0.007714919556075571, "pred_score": 1.5595874404125596, "gold_score": 1.6, "confidence": 0.4527865472134528, "qid": "risk" }, { "type": "score", "correct": true, "kl": 0.013429665742609965, "brier": 0.009674989189, "pred_score": 1.38791, "gold_score": 1.316667, "confidence": 0.419643, "qid": "urgency" } ] }, { "id": "agent_trace_observability_000045", "workflow": "agent_trace_observability", "decisions": [ { "type": "choice", "correct": true, "kl": 0.04648523274815121, "brier": 0.028429188039000002, "confidence": 0.508178, "qid": "action" }, { "type": "noul", "correct": true, "kl": 0.05960967930813327, "brier": 0.048103334792000026, "confidence": 0.661581, "qid": "needs_review" }, { "type": "choice", "correct": false, "kl": 0.20220261896463176, "brier": 0.152256424794, "confidence": 0.460802, "qid": "outcome" }, { "type": "score", "correct": true, "kl": 0.02565163947002617, "brier": 0.012199602295999989, "pred_score": 0.7187100000000001, "gold_score": 0.733333, "confidence": 0.509248, "qid": "risk" }, { "type": "score", "correct": false, "kl": 0.4883334463640648, "brier": 0.35224931568999995, "pred_score": 0.810365, "gold_score": 0.183333, "confidence": 0.486239, "qid": "urgency" } ] }, { "id": "agent_trace_observability_000046", "workflow": "agent_trace_observability", "decisions": [ { "type": "choice", "correct": true, "kl": 0.13279823081285672, "brier": 0.09629090126599997, "confidence": 0.54793, "qid": "action" }, { "type": "noul", "correct": true, "kl": 0.0037807221554115936, "brier": 0.0015362424500000027, "confidence": 0.875618, "qid": "needs_review" }, { "type": "choice", "correct": true, "kl": 0.1387709026144918, "brier": 0.07645355949199999, "confidence": 0.699518, "qid": "outcome" }, { "type": "score", "correct": true, "kl": 0.09620799951497738, "brier": 0.045297397337999964, "pred_score": 0.306628, "gold_score": 0.093333, "confidence": 0.74648, "qid": "risk" }, { "type": "score", "correct": true, "kl": 0.039203762175783655, "brier": 0.023636053419000002, "pred_score": 0.324495, "gold_score": 0.183333, "confidence": 0.725456, "qid": "urgency" } ] }, { "id": "agent_trace_observability_000047", "workflow": "agent_trace_observability", "decisions": [ { "type": "choice", "correct": true, "kl": 0.03151183704539451, "brier": 0.016638137893999996, "confidence": 0.509289, "qid": "action" }, { "type": "noul", "correct": true, "kl": 0.044535996233585604, "brier": 0.039911035391999986, "confidence": 0.608736, "qid": "needs_review" }, { "type": "choice", "correct": false, "kl": 0.17772987570250576, "brier": 0.13132719983075844, "confidence": 0.41382658617341384, "qid": "outcome" }, { "type": "score", "correct": false, "kl": 0.1557558636609655, "brier": 0.10465162156300001, "pred_score": 0.90153, "gold_score": 0.556667, "confidence": 0.60255, "qid": "risk" }, { "type": "score", "correct": false, "kl": 0.03220351244458603, "brier": 0.019942780364000007, "pred_score": 0.933599, "gold_score": 0.75, "confidence": 0.463787, "qid": "urgency" } ] }, { "id": "agent_trace_observability_000048", "workflow": "agent_trace_observability", "decisions": [ { "type": "choice", "correct": true, "kl": 0.07799841999052819, "brier": 0.04170862615502654, "confidence": 0.46149953850046155, "qid": "action" }, { "type": "noul", "correct": true, "kl": 0.06919926860483691, "brier": 0.05677304257800002, "confidence": 0.648184, "qid": "needs_review" }, { "type": "choice", "correct": false, "kl": 0.08861704273226004, "brier": 0.05062440719200001, "confidence": 0.505604, "qid": "outcome" }, { "type": "score", "correct": true, "kl": 0.0026321806042074723, "brier": 0.0009057648229999993, "pred_score": 0.941263, "gold_score": 0.9, "confidence": 0.619124, "qid": "risk" }, { "type": "score", "correct": true, "kl": 0.017240337974587884, "brier": 0.006891381818000006, "pred_score": 0.983896, "gold_score": 1.016667, "confidence": 0.537722, "qid": "urgency" } ] }, { "id": "agent_trace_observability_000049", "workflow": "agent_trace_observability", "decisions": [ { "type": "choice", "correct": true, "kl": 0.15375066348320077, "brier": 0.08221647974434586, "confidence": 0.37944562055437947, "qid": "action" }, { "type": "noul", "correct": true, "kl": 0.022647122126962904, "brier": 0.021899477761999995, "confidence": 0.552026, "qid": "needs_review" }, { "type": "choice", "correct": true, "kl": 0.06917914198467814, "brier": 0.04242927296600002, "confidence": 0.565722, "qid": "outcome" }, { "type": "score", "correct": true, "kl": 0.031098931802666915, "brier": 0.009555262682, "pred_score": 1.731575, "gold_score": 1.593333, "confidence": 0.552826, "qid": "risk" }, { "type": "score", "correct": false, "kl": 0.16042716977794133, "brier": 0.08477831988265801, "pred_score": 1.1618221618221618, "gold_score": 0.686667, "confidence": 0.4146864146864147, "qid": "urgency" } ] }, { "id": "agent_trace_observability_000050", "workflow": "agent_trace_observability", "decisions": [ { "type": "choice", "correct": true, "kl": 0.004847103998024549, "brier": 0.0011008078500000017, "confidence": 0.459104, "qid": "action" }, { "type": "noul", "correct": true, "kl": 0.005074437876250178, "brier": 0.004981018049999996, "confidence": 0.550095, "qid": "needs_review" }, { "type": "choice", "correct": true, "kl": 0.10116130469598711, "brier": 0.045683775046, "confidence": 0.463121, "qid": "outcome" }, { "type": "score", "correct": true, "kl": 0.021165176088039646, "brier": 0.009698518518999995, "pred_score": 1.542619, "gold_score": 1.4, "confidence": 0.494768, "qid": "risk" }, { "type": "score", "correct": true, "kl": 0.041025856388065635, "brier": 0.019361642880000005, "pred_score": 1.19692, "gold_score": 1.0, "confidence": 0.513941, "qid": "urgency" } ] }, { "id": "agent_trace_observability_000051", "workflow": "agent_trace_observability", "decisions": [ { "type": "choice", "correct": true, "kl": 0.02231459921308622, "brier": 0.015804769336999983, "confidence": 0.433462, "qid": "action" }, { "type": "noul", "correct": true, "kl": 0.018182262837909205, "brier": 0.017059630898000007, "confidence": 0.590976, "qid": "needs_review" }, { "type": "choice", "correct": true, "kl": 0.03719261636605715, "brier": 0.021582551385999987, "confidence": 0.534735, "qid": "outcome" }, { "type": "score", "correct": true, "kl": 0.026070742270386145, "brier": 0.011810611649999999, "pred_score": 0.320031, "gold_score": 0.28, "confidence": 0.729293, "qid": "risk" }, { "type": "score", "correct": false, "kl": 0.10749407475991038, "brier": 0.09038532865708199, "pred_score": 0.7127247127247127, "gold_score": 0.483333, "confidence": 0.43887043887043886, "qid": "urgency" } ] }, { "id": "agent_trace_observability_000052", "workflow": "agent_trace_observability", "decisions": [ { "type": "choice", "correct": false, "kl": 0.17367536893252572, "brier": 0.11322551027500001, "confidence": 0.394453, "qid": "action" }, { "type": "noul", "correct": true, "kl": 0.155227450974791, "brier": 0.14063662195200005, "confidence": 0.544824, "qid": "needs_review" }, { "type": "choice", "correct": true, "kl": 0.07191210623588373, "brier": 0.02750348231700001, "confidence": 0.430035, "qid": "outcome" }, { "type": "score", "correct": false, "kl": 0.04807099610494307, "brier": 0.027395149805006035, "pred_score": 1.3275036724963276, "gold_score": 1.07, "confidence": 0.43575156424843575, "qid": "risk" }, { "type": "score", "correct": true, "kl": 0.0026343089749590832, "brier": 0.0013850447082120155, "pred_score": 0.9252830747169254, "gold_score": 0.916667, "confidence": 0.4182755817244183, "qid": "urgency" } ] }, { "id": "agent_trace_observability_000053", "workflow": "agent_trace_observability", "decisions": [ { "type": "choice", "correct": true, "kl": 0.05828755273462269, "brier": 0.025121152854000007, "confidence": 0.445575, "qid": "action" }, { "type": "noul", "correct": true, "kl": 0.017262977936593538, "brier": 0.017127673361999995, "confidence": 0.507459, "qid": "needs_review" }, { "type": "choice", "correct": false, "kl": 0.23236451636526467, "brier": 0.126158879402, "confidence": 0.484298, "qid": "outcome" }, { "type": "score", "correct": true, "kl": 0.032173811384000384, "brier": 0.019374032296000005, "pred_score": 0.818801, "gold_score": 0.966667, "confidence": 0.571696, "qid": "risk" }, { "type": "score", "correct": true, "kl": 0.01701684786438602, "brier": 0.010320150398000003, "pred_score": 1.043423, "gold_score": 0.966667, "confidence": 0.533054, "qid": "urgency" } ] }, { "id": "agent_trace_observability_000054", "workflow": "agent_trace_observability", "decisions": [ { "type": "choice", "correct": true, "kl": 0.20191656835359595, "brier": 0.08405502450972498, "confidence": 0.4763224763224764, "qid": "action" }, { "type": "noul", "correct": true, "kl": 0.07748649926144169, "brier": 0.07191679825799999, "confidence": 0.560373, "qid": "needs_review" }, { "type": "choice", "correct": false, "kl": 0.05217272056384899, "brier": 0.032126159186, "confidence": 0.451422, "qid": "outcome" }, { "type": "score", "correct": false, "kl": 0.6188473376228312, "brier": 0.42292425676600004, "pred_score": 0.875192, "gold_score": 0.2, "confidence": 0.574384, "qid": "risk" }, { "type": "score", "correct": false, "kl": 0.5325080944937479, "brier": 0.31267700412800004, "pred_score": 0.9208810000000001, "gold_score": 0.233333, "confidence": 0.490673, "qid": "urgency" } ] }, { "id": "agent_trace_observability_000055", "workflow": "agent_trace_observability", "decisions": [ { "type": "choice", "correct": true, "kl": 0.19200203451040226, "brier": 0.06827249093700001, "confidence": 0.564723, "qid": "action" }, { "type": "noul", "correct": true, "kl": 0.14151725281313193, "brier": 0.08681944499999998, "confidence": 0.86835, "qid": "needs_review" }, { "type": "choice", "correct": true, "kl": 0.03475891558075442, "brier": 0.02701462367499998, "confidence": 0.679866, "qid": "outcome" }, { "type": "score", "correct": false, "kl": 0.16387392869074469, "brier": 0.13538234308999997, "pred_score": 0.676656, "gold_score": 0.336667, "confidence": 0.498795, "qid": "risk" }, { "type": "score", "correct": true, "kl": 0.012884019532124306, "brier": 0.008105113098000006, "pred_score": 0.47241900000000003, "gold_score": 0.566667, "confidence": 0.603588, "qid": "urgency" } ] }, { "id": "agent_trace_observability_000056", "workflow": "agent_trace_observability", "decisions": [ { "type": "choice", "correct": true, "kl": 0.1357964770963456, "brier": 0.07644686118600001, "confidence": 0.405457, "qid": "action" }, { "type": "noul", "correct": true, "kl": 0.03603639018552464, "brier": 0.023181781842000003, "confidence": 0.759006, "qid": "needs_review" }, { "type": "choice", "correct": true, "kl": 0.10415077589866353, "brier": 0.049396170998000026, "confidence": 0.532215, "qid": "outcome" }, { "type": "score", "correct": true, "kl": 0.018164690793600183, "brier": 0.004761051601999998, "pred_score": 0.863256, "gold_score": 0.743333, "confidence": 0.560388, "qid": "risk" }, { "type": "score", "correct": true, "kl": 0.018835463147291797, "brier": 0.011653546264, "pred_score": 0.71575, "gold_score": 0.62, "confidence": 0.448345, "qid": "urgency" } ] }, { "id": "agent_trace_observability_000057", "workflow": "agent_trace_observability", "decisions": [ { "type": "choice", "correct": true, "kl": 0.15592473219533048, "brier": 0.06685133701499998, "confidence": 0.510081, "qid": "action" }, { "type": "noul", "correct": true, "kl": 0.016738333874696282, "brier": 0.015933267072, "confidence": 0.577411, "qid": "needs_review" }, { "type": "choice", "correct": true, "kl": 0.02626142118350039, "brier": 0.017417013819, "confidence": 0.482732, "qid": "outcome" }, { "type": "score", "correct": true, "kl": 0.007638786218699193, "brier": 0.004411867398999997, "pred_score": 0.950979, "gold_score": 0.983333, "confidence": 0.551497, "qid": "risk" }, { "type": "score", "correct": true, "kl": 0.10571221071161348, "brier": 0.056789809601999995, "pred_score": 0.913118, "gold_score": 1.216667, "confidence": 0.453938, "qid": "urgency" } ] }, { "id": "agent_trace_observability_000058", "workflow": "agent_trace_observability", "decisions": [ { "type": "choice", "correct": true, "kl": 0.016150745827456146, "brier": 0.008836445288000002, "confidence": 0.456698, "qid": "action" }, { "type": "noul", "correct": false, "kl": 0.0322510670723274, "brier": 0.032119465058, "confidence": 0.553394, "qid": "needs_review" }, { "type": "choice", "correct": true, "kl": 0.08632118120856175, "brier": 0.03265313951, "confidence": 0.5067780000000001, "qid": "outcome" }, { "type": "score", "correct": true, "kl": 0.06754809503724095, "brier": 0.039335485364, "pred_score": 0.9841179999999999, "gold_score": 0.756667, "confidence": 0.562572, "qid": "risk" }, { "type": "score", "correct": true, "kl": 0.06222215432953626, "brier": 0.027666011594, "pred_score": 1.007819, "gold_score": 1.183333, "confidence": 0.501138, "qid": "urgency" } ] }, { "id": "agent_trace_observability_000059", "workflow": "agent_trace_observability", "decisions": [ { "type": "choice", "correct": true, "kl": 0.11250427150158589, "brier": 0.025299230517, "confidence": 0.675465, "qid": "action" }, { "type": "noul", "correct": true, "kl": 0.09104195269468178, "brier": 0.033495449138, "confidence": 0.932746, "qid": "needs_review" }, { "type": "choice", "correct": true, "kl": 0.23105856529189295, "brier": 0.10963860588299998, "confidence": 0.764257, "qid": "outcome" }, { "type": "score", "correct": true, "kl": 0.0575771289994938, "brier": 0.016885874862000015, "pred_score": 2.649087, "gold_score": 2.433333, "confidence": 0.711196, "qid": "risk" }, { "type": "score", "correct": true, "kl": 0.029401517878445957, "brier": 0.006276490443000008, "pred_score": 2.460119, "gold_score": 2.333333, "confidence": 0.5661, "qid": "urgency" } ] }, { "id": "agent_trace_observability_000060", "workflow": "agent_trace_observability", "decisions": [ { "type": "choice", "correct": true, "kl": 0.18776672684736817, "brier": 0.13920704223799998, "confidence": 0.437229, "qid": "action" }, { "type": "noul", "correct": true, "kl": 0.022340474200929647, "brier": 0.010998817928000008, "confidence": 0.829175, "qid": "needs_review" }, { "type": "choice", "correct": true, "kl": 0.018785325496405325, "brier": 0.013337312898000006, "confidence": 0.662677, "qid": "outcome" }, { "type": "score", "correct": true, "kl": 0.0016361308754825872, "brier": 0.00046812324600000035, "pred_score": 0.653707, "gold_score": 0.626667, "confidence": 0.502739, "qid": "risk" }, { "type": "score", "correct": true, "kl": 0.012073045247167274, "brier": 0.003701469259999992, "pred_score": 0.492603, "gold_score": 0.583333, "confidence": 0.586737, "qid": "urgency" } ] }, { "id": "agent_trace_observability_000061", "workflow": "agent_trace_observability", "decisions": [ { "type": "choice", "correct": true, "kl": 0.009125581494527822, "brier": 0.00716467454353257, "confidence": 0.4874524874524875, "qid": "action" }, { "type": "noul", "correct": false, "kl": 0.03088723225326466, "brier": 0.03044179425800002, "confidence": 0.593373, "qid": "needs_review" }, { "type": "choice", "correct": false, "kl": 0.26856369203162694, "brier": 0.16994576941099998, "confidence": 0.441313, "qid": "outcome" }, { "type": "score", "correct": true, "kl": 0.09760905284629814, "brier": 0.034643286401999984, "pred_score": 1.759279, "gold_score": 1.45, "confidence": 0.559322, "qid": "risk" }, { "type": "score", "correct": true, "kl": 0.0029973788332003405, "brier": 0.0010994044341529905, "pred_score": 1.4313584313584315, "gold_score": 1.416667, "confidence": 0.40093240093240096, "qid": "urgency" } ] }, { "id": "agent_trace_observability_000062", "workflow": "agent_trace_observability", "decisions": [ { "type": "choice", "correct": false, "kl": 0.2336094434320642, "brier": 0.12028009079799996, "confidence": 0.469394, "qid": "action" }, { "type": "noul", "correct": true, "kl": 0.10960479212250063, "brier": 0.08794398604999998, "confidence": 0.640305, "qid": "needs_review" }, { "type": "choice", "correct": false, "kl": 0.29162747683314405, "brier": 0.20163652875899998, "confidence": 0.526376, "qid": "outcome" }, { "type": "score", "correct": false, "kl": 0.02053531897239456, "brier": 0.004771158725999997, "pred_score": 0.616588, "gold_score": 0.583333, "confidence": 0.48371, "qid": "risk" }, { "type": "score", "correct": false, "kl": 0.08684242418274289, "brier": 0.05412455439574871, "pred_score": 0.7857592142407859, "gold_score": 0.516667, "confidence": 0.475945524054476, "qid": "urgency" } ] }, { "id": "agent_trace_observability_000063", "workflow": "agent_trace_observability", "decisions": [ { "type": "choice", "correct": true, "kl": 0.015645540005535724, "brier": 0.011194009954999997, "confidence": 0.656573, "qid": "action" }, { "type": "noul", "correct": true, "kl": 0.008955919814709404, "brier": 0.0035379189620000067, "confidence": 0.902059, "qid": "needs_review" }, { "type": "choice", "correct": true, "kl": 0.02860379795539726, "brier": 0.006679687330000002, "confidence": 0.757096, "qid": "outcome" }, { "type": "score", "correct": true, "kl": 0.02875714417829467, "brier": 0.022915804436999997, "pred_score": 2.60151, "gold_score": 2.483333, "confidence": 0.67582, "qid": "risk" }, { "type": "score", "correct": true, "kl": 0.0014741908879764756, "brier": 0.0005692948150000022, "pred_score": 2.375459, "gold_score": 2.383333, "confidence": 0.525344, "qid": "urgency" } ] }, { "id": "agent_trace_observability_000064", "workflow": "agent_trace_observability", "decisions": [ { "type": "choice", "correct": true, "kl": 0.02747830949564643, "brier": 0.01375169343444093, "confidence": 0.3938593938593939, "qid": "action" }, { "type": "noul", "correct": true, "kl": 0.018337399177467975, "brier": 0.01749609592200001, "confidence": 0.573136, "qid": "needs_review" }, { "type": "choice", "correct": true, "kl": 0.012879791200485803, "brier": 0.008376191553999997, "confidence": 0.566882, "qid": "outcome" }, { "type": "score", "correct": true, "kl": 0.05873968872710334, "brier": 0.03556707768999999, "pred_score": 0.604474, "gold_score": 0.37, "confidence": 0.527134, "qid": "risk" }, { "type": "score", "correct": false, "kl": 0.02372185714323985, "brier": 0.01671275271, "pred_score": 0.871074, "gold_score": 0.866667, "confidence": 0.473696, "qid": "urgency" } ] }, { "id": "agent_trace_observability_000065", "workflow": "agent_trace_observability", "decisions": [ { "type": "choice", "correct": true, "kl": 0.039833374352873485, "brier": 0.020570229962000006, "confidence": 0.657861, "qid": "action" }, { "type": "noul", "correct": true, "kl": 0.004134141410374333, "brier": 0.0013579344980000005, "confidence": 0.90061, "qid": "needs_review" }, { "type": "choice", "correct": true, "kl": 0.05568839096132278, "brier": 0.014413403693999998, "confidence": 0.707312, "qid": "outcome" }, { "type": "score", "correct": true, "kl": 0.014529455616192705, "brier": 0.003661516565999997, "pred_score": 2.563824, "gold_score": 2.65, "confidence": 0.648597, "qid": "risk" }, { "type": "score", "correct": true, "kl": 0.00258905032296897, "brier": 0.0016567262289999972, "pred_score": 2.3355889999999997, "gold_score": 2.35, "confidence": 0.47785, "qid": "urgency" } ] }, { "id": "agent_trace_observability_000066", "workflow": "agent_trace_observability", "decisions": [ { "type": "choice", "correct": true, "kl": 0.02817197662673028, "brier": 0.004950084252653888, "confidence": 0.4648605351394649, "qid": "action" }, { "type": "noul", "correct": false, "kl": 0.002077315591863648, "brier": 0.0020766434580000003, "confidence": 0.515556, "qid": "needs_review" }, { "type": "choice", "correct": true, "kl": 0.024258227447909203, "brier": 0.008732378490000001, "confidence": 0.485051, "qid": "outcome" }, { "type": "score", "correct": false, "kl": 0.023118181808367468, "brier": 0.012969637737344476, "pred_score": 0.6186086186086186, "gold_score": 0.683333, "confidence": 0.477997477997478, "qid": "risk" }, { "type": "score", "correct": true, "kl": 0.04401008071752265, "brier": 0.010888969146000006, "pred_score": 0.909024, "gold_score": 0.766667, "confidence": 0.501301, "qid": "urgency" } ] }, { "id": "agent_trace_observability_000067", "workflow": "agent_trace_observability", "decisions": [ { "type": "choice", "correct": true, "kl": 0.02709359071366865, "brier": 0.023074957706000016, "confidence": 0.606234, "qid": "action" }, { "type": "noul", "correct": true, "kl": 0.00010564327598036518, "brier": 3.9019777999999954e-05, "confidence": 0.895583, "qid": "needs_review" }, { "type": "choice", "correct": true, "kl": 0.08652020141656369, "brier": 0.05456964816799997, "confidence": 0.684702, "qid": "outcome" }, { "type": "score", "correct": true, "kl": 0.10132121862899299, "brier": 0.055071650832999995, "pred_score": 0.289765, "gold_score": 0.095, "confidence": 0.74589, "qid": "risk" }, { "type": "score", "correct": true, "kl": 0.005994504140823502, "brier": 0.003094013324999999, "pred_score": 0.27338, "gold_score": 0.24, "confidence": 0.761762, "qid": "urgency" } ] }, { "id": "agent_trace_observability_000068", "workflow": "agent_trace_observability", "decisions": [ { "type": "choice", "correct": false, "kl": 0.03000314321466538, "brier": 0.009785473103229667, "confidence": 0.3942256057743942, "qid": "action" }, { "type": "noul", "correct": true, "kl": 0.008885934560426668, "brier": 0.00873736243199999, "confidence": 0.540571, "qid": "needs_review" }, { "type": "choice", "correct": true, "kl": 0.009145384687978722, "brier": 0.007143347959000003, "confidence": 0.489205, "qid": "outcome" }, { "type": "score", "correct": true, "kl": 0.07785609541591187, "brier": 0.05753148140300001, "pred_score": 0.6922550000000001, "gold_score": 0.43, "confidence": 0.457241, "qid": "risk" }, { "type": "score", "correct": true, "kl": 0.018425169441869314, "brier": 0.009358060732999995, "pred_score": 1.0712789999999999, "gold_score": 1.183333, "confidence": 0.519759, "qid": "urgency" } ] }, { "id": "agent_trace_observability_000069", "workflow": "agent_trace_observability", "decisions": [ { "type": "choice", "correct": false, "kl": 0.1399822291159208, "brier": 0.10560170752399999, "confidence": 0.670005, "qid": "action" }, { "type": "noul", "correct": true, "kl": 0.0016231447564535005, "brier": 0.00039976608799999967, "confidence": 0.929195, "qid": "needs_review" }, { "type": "choice", "correct": true, "kl": 0.08174473105545023, "brier": 0.047286421333999996, "confidence": 0.642368, "qid": "outcome" }, { "type": "score", "correct": true, "kl": 0.01263417574326256, "brier": 0.0023324326499999954, "pred_score": 2.5822590000000005, "gold_score": 2.613333, "confidence": 0.6569200000000001, "qid": "risk" }, { "type": "score", "correct": true, "kl": 0.021747654414961043, "brier": 0.016886209284999995, "pred_score": 2.4300770000000003, "gold_score": 2.55, "confidence": 0.549606, "qid": "urgency" } ] }, { "id": "agent_trace_observability_000070", "workflow": "agent_trace_observability", "decisions": [ { "type": "choice", "correct": true, "kl": 0.06929615088917722, "brier": 0.04072455672690706, "confidence": 0.3830936169063831, "qid": "action" }, { "type": "noul", "correct": true, "kl": 0.029905607235712478, "brier": 0.028670066882000002, "confidence": 0.556938, "qid": "needs_review" }, { "type": "choice", "correct": true, "kl": 0.00635980458398764, "brier": 0.004398380468000007, "confidence": 0.596132, "qid": "outcome" }, { "type": "score", "correct": true, "kl": 0.026180634389788758, "brier": 0.013489764849999999, "pred_score": 1.5203259999999998, "gold_score": 1.69, "confidence": 0.529419, "qid": "risk" }, { "type": "score", "correct": true, "kl": 0.14640922612535456, "brier": 0.06275565204599999, "pred_score": 0.9986079999999999, "gold_score": 1.433333, "confidence": 0.471241, "qid": "urgency" } ] }, { "id": "agent_trace_observability_000071", "workflow": "agent_trace_observability", "decisions": [ { "type": "choice", "correct": false, "kl": 0.1508445303369918, "brier": 0.09211756053799999, "confidence": 0.42975, "qid": "action" }, { "type": "noul", "correct": true, "kl": 0.066975464237055, "brier": 0.052497072391999994, "confidence": 0.671319, "qid": "needs_review" }, { "type": "choice", "correct": false, "kl": 0.10966088868295018, "brier": 0.05894765525769377, "confidence": 0.39395260604739396, "qid": "outcome" }, { "type": "score", "correct": false, "kl": 0.16898894879100648, "brier": 0.10645161111200001, "pred_score": 0.856889, "gold_score": 0.483333, "confidence": 0.560423, "qid": "risk" }, { "type": "score", "correct": false, "kl": 0.18679778602799108, "brier": 0.137469803657, "pred_score": 0.8329169999999999, "gold_score": 0.45, "confidence": 0.501937, "qid": "urgency" } ] }, { "id": "agent_trace_observability_000072", "workflow": "agent_trace_observability", "decisions": [ { "type": "choice", "correct": false, "kl": 0.07586735044671421, "brier": 0.04593523437600003, "confidence": 0.526472, "qid": "action" }, { "type": "noul", "correct": true, "kl": 0.010549305730326544, "brier": 0.00982101125, "confidence": 0.653408, "qid": "needs_review" }, { "type": "choice", "correct": true, "kl": 0.047655698883759756, "brier": 0.018312795745999994, "confidence": 0.514526, "qid": "outcome" }, { "type": "score", "correct": true, "kl": 0.011493276815458482, "brier": 0.003543436322, "pred_score": 1.70454, "gold_score": 1.65, "confidence": 0.538746, "qid": "risk" }, { "type": "score", "correct": false, "kl": 0.038145691584953574, "brier": 0.016318299987547524, "pred_score": 1.443203443203443, "gold_score": 1.366667, "confidence": 0.427024427024427, "qid": "urgency" } ] }, { "id": "agent_trace_observability_000073", "workflow": "agent_trace_observability", "decisions": [ { "type": "choice", "correct": true, "kl": 0.01773312292894825, "brier": 0.010813004307999988, "confidence": 0.446508, "qid": "action" }, { "type": "noul", "correct": true, "kl": 0.012001503508599534, "brier": 0.011709396512, "confidence": 0.550151, "qid": "needs_review" }, { "type": "choice", "correct": false, "kl": 0.3017209421709083, "brier": 0.19126051247400003, "confidence": 0.438792, "qid": "outcome" }, { "type": "score", "correct": true, "kl": 0.06349042102186161, "brier": 0.03796246627800001, "pred_score": 1.414094, "gold_score": 1.676667, "confidence": 0.444756, "qid": "risk" }, { "type": "score", "correct": false, "kl": 0.14486623805251442, "brier": 0.07769349237199999, "pred_score": 1.207946, "gold_score": 1.63, "confidence": 0.466646, "qid": "urgency" } ] }, { "id": "agent_trace_observability_000074", "workflow": "agent_trace_observability", "decisions": [ { "type": "choice", "correct": true, "kl": 0.08649695353984056, "brier": 0.029933588659999993, "confidence": 0.461503, "qid": "action" }, { "type": "noul", "correct": true, "kl": 0.04920599025205387, "brier": 0.02555291817800001, "confidence": 0.803634, "qid": "needs_review" }, { "type": "choice", "correct": true, "kl": 0.08896947374725821, "brier": 0.044435488690000004, "confidence": 0.528358, "qid": "outcome" }, { "type": "score", "correct": true, "kl": 0.0929958768497345, "brier": 0.035976785849999963, "pred_score": 0.43907999999999997, "gold_score": 0.216667, "confidence": 0.62816, "qid": "risk" }, { "type": "score", "correct": true, "kl": 0.227127439947693, "brier": 0.13052309653, "pred_score": 0.5192030000000001, "gold_score": 0.133333, "confidence": 0.577901, "qid": "urgency" } ] }, { "id": "agent_trace_observability_000075", "workflow": "agent_trace_observability", "decisions": [ { "type": "choice", "correct": true, "kl": 0.025309269093431466, "brier": 0.015507770962000003, "confidence": 0.510475, "qid": "action" }, { "type": "noul", "correct": true, "kl": 0.0021392988135443686, "brier": 0.001613233602000001, "confidence": 0.738266, "qid": "needs_review" }, { "type": "choice", "correct": true, "kl": 0.01872462714279885, "brier": 0.0018563168780000002, "confidence": 0.43305, "qid": "outcome" }, { "type": "score", "correct": true, "kl": 0.03542157483188803, "brier": 0.015747039374, "pred_score": 0.828538, "gold_score": 0.683333, "confidence": 0.620434, "qid": "risk" }, { "type": "score", "correct": false, "kl": 0.037510378667246574, "brier": 0.01763566801799999, "pred_score": 0.737572, "gold_score": 0.566667, "confidence": 0.464376, "qid": "urgency" } ] }, { "id": "agent_trace_observability_000076", "workflow": "agent_trace_observability", "decisions": [ { "type": "choice", "correct": true, "kl": 0.00802870705310024, "brier": 0.003087739778999998, "confidence": 0.451836, "qid": "action" }, { "type": "noul", "correct": true, "kl": 0.010177687106773553, "brier": 0.009968155207999992, "confidence": 0.546069, "qid": "needs_review" }, { "type": "choice", "correct": true, "kl": 0.006908400712653605, "brier": 0.004448611157999997, "confidence": 0.530895, "qid": "outcome" }, { "type": "score", "correct": true, "kl": 0.032443381007252414, "brier": 0.01675343348291331, "pred_score": 1.4195935804064197, "gold_score": 1.633333, "confidence": 0.45574154425845576, "qid": "risk" }, { "type": "score", "correct": false, "kl": 0.24767222898987337, "brier": 0.11705878805551066, "pred_score": 1.1480348519651482, "gold_score": 1.733333, "confidence": 0.4505635494364506, "qid": "urgency" } ] }, { "id": "agent_trace_observability_000077", "workflow": "agent_trace_observability", "decisions": [ { "type": "choice", "correct": true, "kl": 0.017263188377699953, "brier": 0.009783870397999996, "confidence": 0.669845, "qid": "action" }, { "type": "noul", "correct": true, "kl": 0.0030007004237802663, "brier": 0.0012978493520000003, "confidence": 0.867859, "qid": "needs_review" }, { "type": "choice", "correct": true, "kl": 0.03053069603647051, "brier": 0.02127268441900001, "confidence": 0.52474, "qid": "outcome" }, { "type": "score", "correct": true, "kl": 0.052358429370210804, "brier": 0.018877811017999995, "pred_score": 2.415501, "gold_score": 2.426667, "confidence": 0.565132, "qid": "risk" }, { "type": "score", "correct": true, "kl": 0.014870818861933012, "brier": 0.0020181040379999996, "pred_score": 2.1359830000000004, "gold_score": 2.233333, "confidence": 0.401141, "qid": "urgency" } ] }, { "id": "agent_trace_observability_000078", "workflow": "agent_trace_observability", "decisions": [ { "type": "choice", "correct": true, "kl": 0.08259530685976324, "brier": 0.031400919679000015, "confidence": 0.516306, "qid": "action" }, { "type": "noul", "correct": true, "kl": 9.187055568764778e-06, "brier": 8.520191999999856e-06, "confidence": 0.635397, "qid": "needs_review" }, { "type": "choice", "correct": false, "kl": 0.1440428714425015, "brier": 0.08762707315244277, "confidence": 0.44921944921944923, "qid": "outcome" }, { "type": "score", "correct": true, "kl": 0.04103922749062396, "brier": 0.011913132017000005, "pred_score": 1.780332, "gold_score": 1.783333, "confidence": 0.569837, "qid": "risk" }, { "type": "score", "correct": false, "kl": 0.08480163691337958, "brier": 0.03178025204653879, "pred_score": 1.4667804667804667, "gold_score": 1.8, "confidence": 0.3982393982393982, "qid": "urgency" } ] }, { "id": "agent_trace_observability_000079", "workflow": "agent_trace_observability", "decisions": [ { "type": "choice", "correct": true, "kl": 0.045535494252103154, "brier": 0.023624277025999986, "confidence": 0.61404, "qid": "action" }, { "type": "noul", "correct": true, "kl": 0.0036234617902553834, "brier": 0.0013041745919999998, "confidence": 0.891131, "qid": "needs_review" }, { "type": "choice", "correct": true, "kl": 0.029479667327233033, "brier": 0.014026044426, "confidence": 0.695813, "qid": "outcome" }, { "type": "score", "correct": false, "kl": 0.004914599045423393, "brier": 0.0011801574077483033, "pred_score": 0.6145013854986144, "gold_score": 0.586667, "confidence": 0.4737495262504737, "qid": "risk" }, { "type": "score", "correct": false, "kl": 0.18653026800871159, "brier": 0.124896584518, "pred_score": 0.405649, "gold_score": 0.806667, "confidence": 0.653493, "qid": "urgency" } ] }, { "id": "agent_trace_observability_000080", "workflow": "agent_trace_observability", "decisions": [ { "type": "choice", "correct": true, "kl": 0.05271811311655066, "brier": 0.03328523980599999, "confidence": 0.444931, "qid": "action" }, { "type": "noul", "correct": true, "kl": 0.10700707847320229, "brier": 0.097385201792, "confidence": 0.562669, "qid": "needs_review" }, { "type": "choice", "correct": false, "kl": 0.10650719081982332, "brier": 0.042083086038, "confidence": 0.426042, "qid": "outcome" }, { "type": "score", "correct": true, "kl": 0.055191729144536315, "brier": 0.018869814202999994, "pred_score": 1.649621, "gold_score": 1.516667, "confidence": 0.490485, "qid": "risk" }, { "type": "score", "correct": false, "kl": 0.12548131723635947, "brier": 0.05690700025800001, "pred_score": 1.335318, "gold_score": 0.916667, "confidence": 0.39721, "qid": "urgency" } ] }, { "id": "agent_trace_observability_000081", "workflow": "agent_trace_observability", "decisions": [ { "type": "choice", "correct": true, "kl": 0.004754216481079041, "brier": 0.002007001454000001, "confidence": 0.455158, "qid": "action" }, { "type": "noul", "correct": false, "kl": 0.013969985387137354, "brier": 0.013946504072000009, "confidence": 0.533506, "qid": "needs_review" }, { "type": "choice", "correct": true, "kl": 0.0053774501174096954, "brier": 0.002241938550000001, "confidence": 0.494287, "qid": "outcome" }, { "type": "score", "correct": true, "kl": 0.04452131621323213, "brier": 0.02536985002799998, "pred_score": 0.9910059999999999, "gold_score": 0.79, "confidence": 0.538584, "qid": "risk" }, { "type": "score", "correct": true, "kl": 0.020088366349439887, "brier": 0.007366420093999996, "pred_score": 1.055735, "gold_score": 1.216667, "confidence": 0.484473, "qid": "urgency" } ] }, { "id": "agent_trace_observability_000082", "workflow": "agent_trace_observability", "decisions": [ { "type": "choice", "correct": true, "kl": 0.046455370398618884, "brier": 0.019422122713999995, "confidence": 0.404608, "qid": "action" }, { "type": "noul", "correct": true, "kl": 0.052304989891595044, "brier": 0.050723673032000004, "confidence": 0.524079, "qid": "needs_review" }, { "type": "choice", "correct": true, "kl": 0.05221890084803406, "brier": 0.04427602934499998, "confidence": 0.55561, "qid": "outcome" }, { "type": "score", "correct": true, "kl": 0.004776645028631877, "brier": 0.0016249483639999992, "pred_score": 0.276864, "gold_score": 0.333333, "confidence": 0.777374, "qid": "risk" }, { "type": "score", "correct": true, "kl": 0.025490624298250452, "brier": 0.01145815949, "pred_score": 0.882876, "gold_score": 1.016667, "confidence": 0.453078, "qid": "urgency" } ] }, { "id": "agent_trace_observability_000083", "workflow": "agent_trace_observability", "decisions": [ { "type": "choice", "correct": true, "kl": 0.15833445206299515, "brier": 0.06593839591096234, "confidence": 0.4849795150204849, "qid": "action" }, { "type": "noul", "correct": true, "kl": 0.12264709254911871, "brier": 0.10391328720000001, "confidence": 0.605393, "qid": "needs_review" }, { "type": "choice", "correct": false, "kl": 0.14332359297466146, "brier": 0.07280641002600002, "confidence": 0.417242, "qid": "outcome" }, { "type": "score", "correct": true, "kl": 0.03952758837079715, "brier": 0.02843130582599998, "pred_score": 0.9107080000000001, "gold_score": 0.99, "confidence": 0.570427, "qid": "risk" }, { "type": "score", "correct": true, "kl": 0.027955409958027217, "brier": 0.02057230430000001, "pred_score": 0.931164, "gold_score": 0.966667, "confidence": 0.451644, "qid": "urgency" } ] }, { "id": "agent_trace_observability_000084", "workflow": "agent_trace_observability", "decisions": [ { "type": "choice", "correct": true, "kl": 0.021681709075931158, "brier": 0.011546386873999993, "confidence": 0.55331, "qid": "action" }, { "type": "noul", "correct": true, "kl": 0.00174127049463536, "brier": 0.0008218268819999976, "confidence": 0.856396, "qid": "needs_review" }, { "type": "choice", "correct": false, "kl": 0.3913602568686504, "brier": 0.32466920168099994, "confidence": 0.71844, "qid": "outcome" }, { "type": "score", "correct": true, "kl": 0.29326117997333834, "brier": 0.202562482986, "pred_score": 0.524545, "gold_score": 0.11, "confidence": 0.569708, "qid": "risk" }, { "type": "score", "correct": true, "kl": 0.08139368654178657, "brier": 0.054284187762999994, "pred_score": 0.444729, "gold_score": 0.213333, "confidence": 0.640818, "qid": "urgency" } ] }, { "id": "agent_trace_observability_000085", "workflow": "agent_trace_observability", "decisions": [ { "type": "choice", "correct": false, "kl": 0.19581797209943483, "brier": 0.095653886558, "confidence": 0.48652, "qid": "action" }, { "type": "noul", "correct": true, "kl": 0.07844143666053713, "brier": 0.06526884499999999, "confidence": 0.636017, "qid": "needs_review" }, { "type": "choice", "correct": true, "kl": 0.08743213454038702, "brier": 0.039856274118, "confidence": 0.459889, "qid": "outcome" }, { "type": "score", "correct": false, "kl": 0.1025848165930306, "brier": 0.08966169258900002, "pred_score": 0.681528, "gold_score": 0.44, "confidence": 0.469639, "qid": "risk" }, { "type": "score", "correct": false, "kl": 0.017950363991491712, "brier": 0.013104545002000005, "pred_score": 0.776701, "gold_score": 0.69, "confidence": 0.464314, "qid": "urgency" } ] }, { "id": "agent_trace_observability_000086", "workflow": "agent_trace_observability", "decisions": [ { "type": "choice", "correct": false, "kl": 0.172438661499198, "brier": 0.11975707208799999, "confidence": 0.664665, "qid": "action" }, { "type": "noul", "correct": true, "kl": 0.0033305261796365664, "brier": 0.0011910128180000028, "confidence": 0.892264, "qid": "needs_review" }, { "type": "choice", "correct": true, "kl": 0.01582856762545369, "brier": 0.008145039211999995, "confidence": 0.689558, "qid": "outcome" }, { "type": "score", "correct": true, "kl": 0.0052321902594287956, "brier": 0.0019865779620000064, "pred_score": 2.4903269999999997, "gold_score": 2.556667, "confidence": 0.596289, "qid": "risk" }, { "type": "score", "correct": true, "kl": 0.01077589091723831, "brier": 0.005733016638000004, "pred_score": 2.2550090000000003, "gold_score": 2.366667, "confidence": 0.435704, "qid": "urgency" } ] }, { "id": "agent_trace_observability_000087", "workflow": "agent_trace_observability", "decisions": [ { "type": "choice", "correct": true, "kl": 0.0319567151281303, "brier": 0.013743429117761713, "confidence": 0.4702365297634703, "qid": "action" }, { "type": "noul", "correct": true, "kl": 0.00020440449265874717, "brier": 0.0001799304499999987, "confidence": 0.676152, "qid": "needs_review" }, { "type": "choice", "correct": true, "kl": 0.05941084607080066, "brier": 0.017742477485999997, "confidence": 0.568827, "qid": "outcome" }, { "type": "score", "correct": true, "kl": 0.08622458847776898, "brier": 0.06209269438600001, "pred_score": 0.506084, "gold_score": 0.266667, "confidence": 0.57375, "qid": "risk" }, { "type": "score", "correct": true, "kl": 0.16726712448622508, "brier": 0.07357168052799999, "pred_score": 0.717584, "gold_score": 1.033333, "confidence": 0.437477, "qid": "urgency" } ] }, { "id": "agent_trace_observability_000088", "workflow": "agent_trace_observability", "decisions": [ { "type": "choice", "correct": true, "kl": 0.08152992124311842, "brier": 0.05953348906299999, "confidence": 0.61991, "qid": "action" }, { "type": "noul", "correct": true, "kl": 0.004566947823726058, "brier": 0.0016500214579999966, "confidence": 0.908723, "qid": "needs_review" }, { "type": "choice", "correct": true, "kl": 0.06993260814512894, "brier": 0.027310243655999995, "confidence": 0.591491, "qid": "outcome" }, { "type": "score", "correct": true, "kl": 0.02835252716071536, "brier": 0.0024918694459999994, "pred_score": 2.511191, "gold_score": 2.616667, "confidence": 0.630533, "qid": "risk" }, { "type": "score", "correct": true, "kl": 0.006628368226799547, "brier": 0.0027179245730000003, "pred_score": 2.345466, "gold_score": 2.433333, "confidence": 0.522016, "qid": "urgency" } ] }, { "id": "agent_trace_observability_000089", "workflow": "agent_trace_observability", "decisions": [ { "type": "choice", "correct": true, "kl": 0.008128120723808856, "brier": 0.0014247374660000002, "confidence": 0.392185, "qid": "action" }, { "type": "noul", "correct": true, "kl": 0.0007155565494308645, "brier": 0.0007142688079999999, "confidence": 0.514435, "qid": "needs_review" }, { "type": "choice", "correct": true, "kl": 0.02852214147923534, "brier": 0.011281720094000006, "confidence": 0.501911, "qid": "outcome" }, { "type": "score", "correct": true, "kl": 0.056440630756859726, "brier": 0.03001101535799998, "pred_score": 0.56935, "gold_score": 0.366667, "confidence": 0.56035, "qid": "risk" }, { "type": "score", "correct": false, "kl": 0.2584044679566109, "brier": 0.16091256318935407, "pred_score": 1.0713259286740715, "gold_score": 0.533333, "confidence": 0.45891854108145896, "qid": "urgency" } ] }, { "id": "agent_trace_observability_000090", "workflow": "agent_trace_observability", "decisions": [ { "type": "choice", "correct": true, "kl": 0.10566515767403978, "brier": 0.06157720028599998, "confidence": 0.584532, "qid": "action" }, { "type": "noul", "correct": true, "kl": 0.020795528502659748, "brier": 0.010808616392000012, "confidence": 0.819819, "qid": "needs_review" }, { "type": "choice", "correct": true, "kl": 0.21730997578667238, "brier": 0.14825993213930802, "confidence": 0.4415135584864416, "qid": "outcome" }, { "type": "score", "correct": true, "kl": 0.03411183032355695, "brier": 0.013568619577999987, "pred_score": 0.364261, "gold_score": 0.223333, "confidence": 0.697443, "qid": "risk" }, { "type": "score", "correct": true, "kl": 0.12185046560799025, "brier": 0.07333973304100003, "pred_score": 0.498584, "gold_score": 0.2, "confidence": 0.599069, "qid": "urgency" } ] }, { "id": "agent_trace_observability_000091", "workflow": "agent_trace_observability", "decisions": [ { "type": "choice", "correct": true, "kl": 0.016498457316708955, "brier": 0.009090155426000014, "confidence": 0.635844, "qid": "action" }, { "type": "noul", "correct": true, "kl": 0.015132695433322706, "brier": 0.0058143014480000026, "confidence": 0.872749, "qid": "needs_review" }, { "type": "choice", "correct": true, "kl": 0.027558680324754174, "brier": 0.0036150209099999994, "confidence": 0.772666, "qid": "outcome" }, { "type": "score", "correct": true, "kl": 0.010808290510075234, "brier": 0.00038014986999999846, "pred_score": 2.604989, "gold_score": 2.643333, "confidence": 0.673954, "qid": "risk" }, { "type": "score", "correct": true, "kl": 0.048390018162813506, "brier": 0.021813025199999992, "pred_score": 2.382531, "gold_score": 2.26, "confidence": 0.543655, "qid": "urgency" } ] }, { "id": "agent_trace_observability_000092", "workflow": "agent_trace_observability", "decisions": [ { "type": "choice", "correct": false, "kl": 0.4500228737800712, "brier": 0.21917923081399998, "confidence": 0.406631, "qid": "action" }, { "type": "noul", "correct": true, "kl": 0.2531768986424695, "brier": 0.22422225779999994, "confidence": 0.52517, "qid": "needs_review" }, { "type": "choice", "correct": true, "kl": 0.11567572081355616, "brier": 0.034539372580198034, "confidence": 0.3984036015963984, "qid": "outcome" }, { "type": "score", "correct": true, "kl": 0.11750531268738187, "brier": 0.048898023285, "pred_score": 1.6293579999999999, "gold_score": 1.25, "confidence": 0.504652, "qid": "risk" }, { "type": "score", "correct": false, "kl": 0.1020618447362045, "brier": 0.03847188021800001, "pred_score": 1.121468, "gold_score": 0.733333, "confidence": 0.393141, "qid": "urgency" } ] }, { "id": "agent_trace_observability_000093", "workflow": "agent_trace_observability", "decisions": [ { "type": "choice", "correct": true, "kl": 0.021413226801276622, "brier": 0.011963284237000008, "confidence": 0.494848, "qid": "action" }, { "type": "noul", "correct": true, "kl": 0.008599965634259021, "brier": 0.007901502050000005, "confidence": 0.620478, "qid": "needs_review" }, { "type": "choice", "correct": false, "kl": 0.18659222449374133, "brier": 0.10603168552400002, "confidence": 0.465576, "qid": "outcome" }, { "type": "score", "correct": true, "kl": 0.00279804915250382, "brier": 0.0013579376180000021, "pred_score": 0.715147, "gold_score": 0.683333, "confidence": 0.48684, "qid": "risk" }, { "type": "score", "correct": true, "kl": 0.00340592249354325, "brier": 0.0012281418370000013, "pred_score": 0.796512, "gold_score": 0.75, "confidence": 0.462847, "qid": "urgency" } ] }, { "id": "agent_trace_observability_000094", "workflow": "agent_trace_observability", "decisions": [ { "type": "choice", "correct": false, "kl": 0.2933357298907332, "brier": 0.15891400851, "confidence": 0.421755, "qid": "action" }, { "type": "noul", "correct": false, "kl": 0.03355957623566613, "brier": 0.033305740231999996, "confidence": 0.512379, "qid": "needs_review" }, { "type": "choice", "correct": false, "kl": 0.22265048819937924, "brier": 0.10172138897000002, "confidence": 0.42202, "qid": "outcome" }, { "type": "score", "correct": false, "kl": 0.3938626273913109, "brier": 0.24842903036300001, "pred_score": 0.936418, "gold_score": 0.3, "confidence": 0.475066, "qid": "risk" }, { "type": "score", "correct": false, "kl": 0.3398243356058657, "brier": 0.16605307776300002, "pred_score": 1.114094, "gold_score": 0.5, "confidence": 0.494179, "qid": "urgency" } ] }, { "id": "agent_trace_observability_000095", "workflow": "agent_trace_observability", "decisions": [ { "type": "choice", "correct": false, "kl": 0.10924229805744208, "brier": 0.04840786205320652, "confidence": 0.4629765370234629, "qid": "action" }, { "type": "noul", "correct": true, "kl": 0.05246324498861038, "brier": 0.02882352480200002, "confidence": 0.789951, "qid": "needs_review" }, { "type": "choice", "correct": true, "kl": 0.07732296228314604, "brier": 0.043419487247999995, "confidence": 0.681994, "qid": "outcome" }, { "type": "score", "correct": true, "kl": 0.044837207397239474, "brier": 0.008066689621999997, "pred_score": 0.961349, "gold_score": 0.786667, "confidence": 0.58209, "qid": "risk" }, { "type": "score", "correct": false, "kl": 0.141555400558982, "brier": 0.09803464470999997, "pred_score": 0.702273, "gold_score": 1.016667, "confidence": 0.480817, "qid": "urgency" } ] }, { "id": "agent_trace_observability_000096", "workflow": "agent_trace_observability", "decisions": [ { "type": "choice", "correct": true, "kl": 0.03914832293230276, "brier": 0.029086136366, "confidence": 0.617607, "qid": "action" }, { "type": "noul", "correct": true, "kl": 0.0049013403827926245, "brier": 0.0015658727219999952, "confidence": 0.921314, "qid": "needs_review" }, { "type": "choice", "correct": true, "kl": 0.052247553580166514, "brier": 0.032142208010000006, "confidence": 0.728704, "qid": "outcome" }, { "type": "score", "correct": true, "kl": 0.02274496505176697, "brier": 0.007011406598000007, "pred_score": 2.591054, "gold_score": 2.516667, "confidence": 0.664081, "qid": "risk" }, { "type": "score", "correct": true, "kl": 0.035226682866703704, "brier": 0.013347522556999988, "pred_score": 2.397606, "gold_score": 2.2, "confidence": 0.533365, "qid": "urgency" } ] }, { "id": "agent_trace_observability_000097", "workflow": "agent_trace_observability", "decisions": [ { "type": "choice", "correct": true, "kl": 0.08414849841906266, "brier": 0.02384539068916962, "confidence": 0.4360834360834361, "qid": "action" }, { "type": "noul", "correct": true, "kl": 0.032130662422667725, "brier": 0.031106664737999994, "confidence": 0.541954, "qid": "needs_review" }, { "type": "choice", "correct": true, "kl": 0.05058104376243716, "brier": 0.039131354065, "confidence": 0.472242, "qid": "outcome" }, { "type": "score", "correct": false, "kl": 0.053146001236888854, "brier": 0.026456002232999987, "pred_score": 0.7942, "gold_score": 0.7, "confidence": 0.487965, "qid": "risk" }, { "type": "score", "correct": true, "kl": 0.037109894071456574, "brier": 0.022319381792999995, "pred_score": 1.043413, "gold_score": 0.9, "confidence": 0.51337, "qid": "urgency" } ] }, { "id": "agent_trace_observability_000098", "workflow": "agent_trace_observability", "decisions": [ { "type": "choice", "correct": true, "kl": 0.007370361419023982, "brier": 0.004463183360753656, "confidence": 0.4278174278174278, "qid": "action" }, { "type": "noul", "correct": true, "kl": 0.0021027097677532927, "brier": 0.0020626179920000005, "confidence": 0.557886, "qid": "needs_review" }, { "type": "choice", "correct": true, "kl": 0.021194164857924726, "brier": 0.0071933986389999946, "confidence": 0.530019, "qid": "outcome" }, { "type": "score", "correct": true, "kl": 0.08025187147042738, "brier": 0.026460741991000007, "pred_score": 1.1027, "gold_score": 0.783333, "confidence": 0.464537, "qid": "risk" }, { "type": "score", "correct": true, "kl": 0.03050600774151977, "brier": 0.010882169734, "pred_score": 1.342615, "gold_score": 1.2, "confidence": 0.483374, "qid": "urgency" } ] }, { "id": "agent_trace_observability_000099", "workflow": "agent_trace_observability", "decisions": [ { "type": "choice", "correct": true, "kl": 0.03720174346639428, "brier": 0.015897819121999997, "confidence": 0.532099, "qid": "action" }, { "type": "noul", "correct": true, "kl": 0.00015765043957026957, "brier": 0.00014128804999999985, "confidence": 0.658262, "qid": "needs_review" }, { "type": "choice", "correct": true, "kl": 0.06402845161988423, "brier": 0.028918670709999986, "confidence": 0.514902, "qid": "outcome" }, { "type": "score", "correct": true, "kl": 0.038203969128774856, "brier": 0.008698081188, "pred_score": 1.750585, "gold_score": 1.683333, "confidence": 0.556006, "qid": "risk" }, { "type": "score", "correct": true, "kl": 0.008540894671023695, "brier": 0.002251596661000002, "pred_score": 1.46536, "gold_score": 1.533333, "confidence": 0.421877, "qid": "urgency" } ] }, { "id": "customer_service_000000", "workflow": "customer_service", "decisions": [ { "type": "choice", "correct": false, "kl": 0.25588628558841536, "brier": 0.132652148588, "confidence": 0.3774, "qid": "action" }, { "type": "choice", "correct": true, "kl": 0.021277336106000612, "brier": 0.0025904957699999964, "confidence": 0.896973, "qid": "category" }, { "type": "score", "correct": true, "kl": 0.04304040178807267, "brier": 0.017785548326000003, "pred_score": 1.022087, "gold_score": 1.066667, "confidence": 0.628714, "qid": "churn_risk" }, { "type": "noul", "correct": false, "kl": 0.004860850016701258, "brier": 0.004858219591999994, "confidence": 0.515953, "qid": "needs_human" }, { "type": "score", "correct": true, "kl": 0.11606138547393222, "brier": 0.069703256858, "pred_score": 1.488839, "gold_score": 1.2, "confidence": 0.53869, "qid": "urgency" } ] }, { "id": "customer_service_000001", "workflow": "customer_service", "decisions": [ { "type": "choice", "correct": false, "kl": 0.12276867433912704, "brier": 0.054341901262999995, "confidence": 0.439003, "qid": "action" }, { "type": "choice", "correct": true, "kl": 0.011389921513894594, "brier": 0.00046761048299999886, "confidence": 0.897369, "qid": "category" }, { "type": "score", "correct": true, "kl": 0.11238626026731031, "brier": 0.05283341940900001, "pred_score": 1.076878, "gold_score": 1.383333, "confidence": 0.5764900000000001, "qid": "churn_risk" }, { "type": "noul", "correct": true, "kl": 1.73378697453405e-05, "brier": 1.7099551999999468e-05, "confidence": 0.559591, "qid": "needs_human" }, { "type": "score", "correct": false, "kl": 0.23326095141168504, "brier": 0.09358625143899998, "pred_score": 1.608589, "gold_score": 2.1, "confidence": 0.441256, "qid": "urgency" } ] }, { "id": "customer_service_000002", "workflow": "customer_service", "decisions": [ { "type": "choice", "correct": false, "kl": 0.02730783146772009, "brier": 0.012089655288925283, "confidence": 0.33119433119433117, "qid": "action" }, { "type": "choice", "correct": true, "kl": 0.019739233064090037, "brier": 0.000716579675000003, "confidence": 0.951776, "qid": "category" }, { "type": "score", "correct": true, "kl": 0.055742303009561005, "brier": 0.018180846145999997, "pred_score": 1.0944630000000002, "gold_score": 1.016667, "confidence": 0.533344, "qid": "churn_risk" }, { "type": "noul", "correct": false, "kl": 0.023830608216792934, "brier": 0.023756332337999987, "confidence": 0.525654, "qid": "needs_human" }, { "type": "score", "correct": true, "kl": 0.09531939713016929, "brier": 0.058538618616000015, "pred_score": 1.5400390000000002, "gold_score": 1.25, "confidence": 0.461754, "qid": "urgency" } ] }, { "id": "customer_service_000003", "workflow": "customer_service", "decisions": [ { "type": "choice", "correct": false, "kl": 0.29673492741990426, "brier": 0.16609853437900002, "confidence": 0.480621, "qid": "action" }, { "type": "choice", "correct": true, "kl": 0.02581040952242632, "brier": 0.001461308044, "confidence": 0.939359, "qid": "category" }, { "type": "score", "correct": true, "kl": 0.02713080805201213, "brier": 0.01853895669400001, "pred_score": 1.871586, "gold_score": 1.85, "confidence": 0.438786, "qid": "churn_risk" }, { "type": "noul", "correct": true, "kl": 0.008804295304677225, "brier": 0.008487481472000001, "confidence": 0.615144, "qid": "needs_human" }, { "type": "score", "correct": true, "kl": 0.21130342697134613, "brier": 0.14401035814899998, "pred_score": 2.698701, "gold_score": 2.3, "confidence": 0.7521, "qid": "urgency" } ] }, { "id": "customer_service_000004", "workflow": "customer_service", "decisions": [ { "type": "choice", "correct": true, "kl": 0.04517985014317227, "brier": 0.005068412029999999, "confidence": 0.799895, "qid": "action" }, { "type": "choice", "correct": true, "kl": 0.013682558167303398, "brier": 0.002408854414999994, "confidence": 0.921915, "qid": "category" }, { "type": "score", "correct": true, "kl": 0.00520906768374682, "brier": 0.0011311623749999972, "pred_score": 1.970899, "gold_score": 2.0, "confidence": 0.588121, "qid": "churn_risk" }, { "type": "noul", "correct": true, "kl": 0.024520245380589, "brier": 0.016120178568000007, "confidence": 0.760222, "qid": "needs_human" }, { "type": "score", "correct": false, "kl": 0.23580881320375902, "brier": 0.11487002673800005, "pred_score": 2.080776, "gold_score": 1.583333, "confidence": 0.502873, "qid": "urgency" } ] }, { "id": "customer_service_000005", "workflow": "customer_service", "decisions": [ { "type": "choice", "correct": true, "kl": 0.25498703068691314, "brier": 0.0994876838547845, "confidence": 0.36622563377436623, "qid": "action" }, { "type": "choice", "correct": true, "kl": 0.06322611196002581, "brier": 0.006786778014000007, "confidence": 0.863066, "qid": "category" }, { "type": "score", "correct": true, "kl": 0.04966564558770799, "brier": 0.03099277623499999, "pred_score": 1.950198, "gold_score": 2.033333, "confidence": 0.466812, "qid": "churn_risk" }, { "type": "noul", "correct": true, "kl": 0.13556268004998687, "brier": 0.119906962632, "confidence": 0.571813, "qid": "needs_human" }, { "type": "score", "correct": true, "kl": 0.05262734835096865, "brier": 0.028247076727000015, "pred_score": 2.748238, "gold_score": 2.65, "confidence": 0.7967420000000001, "qid": "urgency" } ] }, { "id": "customer_service_000006", "workflow": "customer_service", "decisions": [ { "type": "choice", "correct": true, "kl": 0.020055248378884824, "brier": 0.014523195334000002, "confidence": 0.57596, "qid": "action" }, { "type": "choice", "correct": true, "kl": 0.01154634662200053, "brier": 0.0013841005950000025, "confidence": 0.922684, "qid": "category" }, { "type": "score", "correct": true, "kl": 0.09687548378669184, "brier": 0.03135931906202526, "pred_score": 1.4793934793934793, "gold_score": 1.733333, "confidence": 0.48313148313148313, "qid": "churn_risk" }, { "type": "noul", "correct": true, "kl": 0.00047837394570502366, "brier": 0.00037335513800000044, "confidence": 0.72967, "qid": "needs_human" }, { "type": "score", "correct": true, "kl": 0.04682828806483904, "brier": 0.010999348062999998, "pred_score": 1.913481, "gold_score": 2.1, "confidence": 0.538714, "qid": "urgency" } ] }, { "id": "customer_service_000007", "workflow": "customer_service", "decisions": [ { "type": "choice", "correct": true, "kl": 0.19944997906706663, "brier": 0.07998451544399998, "confidence": 0.359697, "qid": "action" }, { "type": "choice", "correct": true, "kl": 0.08566044639564917, "brier": 0.006105693935999998, "confidence": 0.823044, "qid": "category" }, { "type": "score", "correct": true, "kl": 0.01673264274239606, "brier": 0.004832423754999998, "pred_score": 2.782836, "gold_score": 2.863333, "confidence": 0.8268260000000001, "qid": "churn_risk" }, { "type": "noul", "correct": true, "kl": 0.03694525419791393, "brier": 0.03272449445, "confidence": 0.622085, "qid": "needs_human" }, { "type": "score", "correct": false, "kl": 0.029458349756781313, "brier": 0.016872897065999985, "pred_score": 2.369684, "gold_score": 2.45, "confidence": 0.49036, "qid": "urgency" } ] }, { "id": "customer_service_000008", "workflow": "customer_service", "decisions": [ { "type": "choice", "correct": false, "kl": 0.04691109902344821, "brier": 0.015975720534000006, "confidence": 0.472439, "qid": "action" }, { "type": "choice", "correct": false, "kl": 0.5734812842588245, "brier": 0.4731134521830001, "confidence": 0.6071610000000001, "qid": "category" }, { "type": "score", "correct": false, "kl": 0.030240978866664506, "brier": 0.0140503557, "pred_score": 1.543017, "gold_score": 1.45, "confidence": 0.501931, "qid": "churn_risk" }, { "type": "noul", "correct": true, "kl": 0.016790803651488606, "brier": 0.01465985645, "confidence": 0.647718, "qid": "needs_human" }, { "type": "score", "correct": true, "kl": 0.041593203072019115, "brier": 0.025897662573999995, "pred_score": 2.056167, "gold_score": 2.07, "confidence": 0.609982, "qid": "urgency" } ] }, { "id": "customer_service_000009", "workflow": "customer_service", "decisions": [ { "type": "choice", "correct": true, "kl": 0.026307698421749487, "brier": 0.015902842095, "confidence": 0.45177, "qid": "action" }, { "type": "choice", "correct": false, "kl": 0.1454970884920246, "brier": 0.111407093471, "confidence": 0.670065, "qid": "category" }, { "type": "score", "correct": true, "kl": 0.010384472146802652, "brier": 0.00550936532953168, "pred_score": 2.258998741001259, "gold_score": 2.276667, "confidence": 0.4560945439054561, "qid": "churn_risk" }, { "type": "noul", "correct": true, "kl": 0.03507508919736156, "brier": 0.025464828487999996, "confidence": 0.720495, "qid": "needs_human" }, { "type": "score", "correct": true, "kl": 0.036771361826286784, "brier": 0.007859501010999994, "pred_score": 2.109515, "gold_score": 2.25, "confidence": 0.547166, "qid": "urgency" } ] }, { "id": "customer_service_000010", "workflow": "customer_service", "decisions": [ { "type": "choice", "correct": true, "kl": 0.026718976862290487, "brier": 0.006709216383999994, "confidence": 0.57152, "qid": "action" }, { "type": "choice", "correct": true, "kl": 0.055532217383002805, "brier": 0.007362509574000001, "confidence": 0.906006, "qid": "category" }, { "type": "score", "correct": true, "kl": 0.03334941209011863, "brier": 0.005287828418000002, "pred_score": 1.89722, "gold_score": 2.033333, "confidence": 0.61473, "qid": "churn_risk" }, { "type": "noul", "correct": true, "kl": 0.005380199218668585, "brier": 0.0037939018320000068, "confidence": 0.756446, "qid": "needs_human" }, { "type": "score", "correct": true, "kl": 0.057406831117288476, "brier": 0.03377454923500002, "pred_score": 1.828435, "gold_score": 1.883333, "confidence": 0.612562, "qid": "urgency" } ] }, { "id": "customer_service_000011", "workflow": "customer_service", "decisions": [ { "type": "choice", "correct": false, "kl": 0.25507783769492404, "brier": 0.12171683856599999, "confidence": 0.398865, "qid": "action" }, { "type": "choice", "correct": true, "kl": 0.09309309815757336, "brier": 0.03061478264199998, "confidence": 0.780799, "qid": "category" }, { "type": "score", "correct": true, "kl": 0.013835093163000828, "brier": 0.0031794704950000056, "pred_score": 1.707743, "gold_score": 1.766667, "confidence": 0.502841, "qid": "churn_risk" }, { "type": "noul", "correct": true, "kl": 0.013479228450130817, "brier": 0.013117788338000001, "confidence": 0.552346, "qid": "needs_human" }, { "type": "score", "correct": true, "kl": 0.04181979677775895, "brier": 0.0021221247099999954, "pred_score": 2.792231, "gold_score": 2.866667, "confidence": 0.839501, "qid": "urgency" } ] }, { "id": "customer_service_000012", "workflow": "customer_service", "decisions": [ { "type": "choice", "correct": true, "kl": 0.21872447977058757, "brier": 0.09313959348600001, "confidence": 0.574678, "qid": "action" }, { "type": "choice", "correct": true, "kl": 0.04709375465423703, "brier": 0.006662787425999994, "confidence": 0.900741, "qid": "category" }, { "type": "score", "correct": true, "kl": 0.00885513305368315, "brier": 0.004335004118000008, "pred_score": 1.7895420000000002, "gold_score": 1.783333, "confidence": 0.672112, "qid": "churn_risk" }, { "type": "noul", "correct": true, "kl": 0.00024894529630747723, "brier": 0.000184896450000001, "confidence": 0.750385, "qid": "needs_human" }, { "type": "score", "correct": true, "kl": 0.010088774829914776, "brier": 0.004439312473999996, "pred_score": 1.704987, "gold_score": 1.733333, "confidence": 0.551714, "qid": "urgency" } ] }, { "id": "customer_service_000013", "workflow": "customer_service", "decisions": [ { "type": "choice", "correct": false, "kl": 0.04908231459883323, "brier": 0.023744470705999995, "confidence": 0.411547, "qid": "action" }, { "type": "choice", "correct": true, "kl": 0.006471160602429594, "brier": 0.0006034888199999991, "confidence": 0.890203, "qid": "category" }, { "type": "score", "correct": false, "kl": 0.03541221701112164, "brier": 0.017184390266000008, "pred_score": 1.430988, "gold_score": 1.333333, "confidence": 0.385272, "qid": "churn_risk" }, { "type": "noul", "correct": false, "kl": 0.02590525163957212, "brier": 0.025762230050000008, "confidence": 0.563495, "qid": "needs_human" }, { "type": "score", "correct": false, "kl": 0.3803372216783006, "brier": 0.236480780282, "pred_score": 1.131333, "gold_score": 1.383333, "confidence": 0.648497, "qid": "urgency" } ] }, { "id": "customer_service_000014", "workflow": "customer_service", "decisions": [ { "type": "choice", "correct": true, "kl": 0.09487278999988401, "brier": 0.051681034859000004, "confidence": 0.57109, "qid": "action" }, { "type": "choice", "correct": false, "kl": 0.3653640215997784, "brier": 0.28249622264400004, "confidence": 0.7595500000000001, "qid": "category" }, { "type": "score", "correct": true, "kl": 0.04012346259900107, "brier": 0.022998230095999994, "pred_score": 1.500918, "gold_score": 1.333333, "confidence": 0.454723, "qid": "churn_risk" }, { "type": "noul", "correct": true, "kl": 0.06727230177436097, "brier": 0.05886703968799999, "confidence": 0.611771, "qid": "needs_human" }, { "type": "score", "correct": true, "kl": 0.012103745648796821, "brier": 0.007895226266000007, "pred_score": 2.568497, "gold_score": 2.663333, "confidence": 0.645512, "qid": "urgency" } ] }, { "id": "customer_service_000015", "workflow": "customer_service", "decisions": [ { "type": "choice", "correct": true, "kl": 0.12785386135606913, "brier": 0.06208909478199998, "confidence": 0.582406, "qid": "action" }, { "type": "choice", "correct": true, "kl": 0.041670394082845424, "brier": 0.0053789016410000015, "confidence": 0.919379, "qid": "category" }, { "type": "score", "correct": true, "kl": 0.08623726308084727, "brier": 0.052234084165, "pred_score": 1.486929, "gold_score": 1.203333, "confidence": 0.444596, "qid": "churn_risk" }, { "type": "noul", "correct": true, "kl": 0.05160898758485716, "brier": 0.03587073985800001, "confidence": 0.726077, "qid": "needs_human" }, { "type": "score", "correct": true, "kl": 0.026884714839247557, "brier": 0.004601773651999992, "pred_score": 2.528968, "gold_score": 2.433333, "confidence": 0.618926, "qid": "urgency" } ] }, { "id": "customer_service_000016", "workflow": "customer_service", "decisions": [ { "type": "choice", "correct": true, "kl": 0.12897967244055608, "brier": 0.044636537791, "confidence": 0.564687, "qid": "action" }, { "type": "choice", "correct": true, "kl": 0.009805726749948427, "brier": 0.000200452107, "confidence": 0.97966, "qid": "category" }, { "type": "score", "correct": false, "kl": 0.36204768659194375, "brier": 0.224290760814, "pred_score": 0.990867, "gold_score": 0.463333, "confidence": 0.601754, "qid": "churn_risk" }, { "type": "noul", "correct": true, "kl": 0.0837375154976819, "brier": 0.064571242248, "confidence": 0.670318, "qid": "needs_human" }, { "type": "score", "correct": true, "kl": 0.03471906658688464, "brier": 0.016846252615999986, "pred_score": 2.0387500000000003, "gold_score": 2.12, "confidence": 0.5958990000000001, "qid": "urgency" } ] }, { "id": "customer_service_000017", "workflow": "customer_service", "decisions": [ { "type": "choice", "correct": true, "kl": 0.07587760245456113, "brier": 0.022914971934999993, "confidence": 0.742222, "qid": "action" }, { "type": "choice", "correct": true, "kl": 0.05801481193022237, "brier": 0.012743553338999998, "confidence": 0.85434, "qid": "category" }, { "type": "score", "correct": true, "kl": 0.027118188234123583, "brier": 0.01750401757600001, "pred_score": 2.635172, "gold_score": 2.746667, "confidence": 0.700476, "qid": "churn_risk" }, { "type": "noul", "correct": true, "kl": 0.0013087999166915137, "brier": 0.0005704766419999978, "confidence": 0.869778, "qid": "needs_human" }, { "type": "score", "correct": true, "kl": 0.0023993319728129203, "brier": 0.0003204009540000005, "pred_score": 2.631395, "gold_score": 2.61, "confidence": 0.702335, "qid": "urgency" } ] }, { "id": "customer_service_000018", "workflow": "customer_service", "decisions": [ { "type": "choice", "correct": true, "kl": 0.22547175278273204, "brier": 0.08558956911980241, "confidence": 0.472967527032473, "qid": "action" }, { "type": "choice", "correct": true, "kl": 0.06038583094554441, "brier": 0.0047089944070000025, "confidence": 0.851958, "qid": "category" }, { "type": "score", "correct": true, "kl": 0.19072045800418014, "brier": 0.12868780270199998, "pred_score": 2.477666, "gold_score": 2.836667, "confidence": 0.588192, "qid": "churn_risk" }, { "type": "noul", "correct": true, "kl": 0.011670056861012162, "brier": 0.009507377618000003, "confidence": 0.691053, "qid": "needs_human" }, { "type": "score", "correct": true, "kl": 0.004619914452208284, "brier": 0.0010463034119999996, "pred_score": 2.7664130000000005, "gold_score": 2.766667, "confidence": 0.824455, "qid": "urgency" } ] }, { "id": "customer_service_000019", "workflow": "customer_service", "decisions": [ { "type": "choice", "correct": true, "kl": 0.020559399984173284, "brier": 0.0017789297040000011, "confidence": 0.616831, "qid": "action" }, { "type": "choice", "correct": true, "kl": 0.004945026030609368, "brier": 0.0007517325799999968, "confidence": 0.911568, "qid": "category" }, { "type": "score", "correct": true, "kl": 0.019081683006316158, "brier": 0.012917153333999986, "pred_score": 2.159345, "gold_score": 2.066667, "confidence": 0.538425, "qid": "churn_risk" }, { "type": "noul", "correct": true, "kl": 0.0010317936171393254, "brier": 0.000746370247999998, "confidence": 0.769318, "qid": "needs_human" }, { "type": "score", "correct": true, "kl": 0.02868642574791458, "brier": 0.013438684825999999, "pred_score": 1.980467, "gold_score": 1.833333, "confidence": 0.598484, "qid": "urgency" } ] }, { "id": "customer_service_000020", "workflow": "customer_service", "decisions": [ { "type": "choice", "correct": true, "kl": 0.07376041912291711, "brier": 0.023024335809999998, "confidence": 0.621775, "qid": "action" }, { "type": "choice", "correct": true, "kl": 0.0352114522475792, "brier": 0.006220406142000006, "confidence": 0.877095, "qid": "category" }, { "type": "score", "correct": true, "kl": 0.04693532876500381, "brier": 0.01542266749400001, "pred_score": 2.5329070000000002, "gold_score": 2.553333, "confidence": 0.606486, "qid": "churn_risk" }, { "type": "noul", "correct": true, "kl": 0.006376292990553943, "brier": 0.0042318160019999915, "confidence": 0.774001, "qid": "needs_human" }, { "type": "score", "correct": true, "kl": 0.030704799341356927, "brier": 0.001678958641999999, "pred_score": 2.078363, "gold_score": 2.063333, "confidence": 0.688106, "qid": "urgency" } ] }, { "id": "customer_service_000021", "workflow": "customer_service", "decisions": [ { "type": "choice", "correct": true, "kl": 0.03543676938688027, "brier": 0.013584549166000003, "confidence": 0.456128, "qid": "action" }, { "type": "choice", "correct": true, "kl": 0.02138366557160292, "brier": 0.0032181877420000074, "confidence": 0.908357, "qid": "category" }, { "type": "score", "correct": true, "kl": 0.04527253341103489, "brier": 0.03176724763400002, "pred_score": 1.558315, "gold_score": 1.75, "confidence": 0.422593, "qid": "churn_risk" }, { "type": "noul", "correct": false, "kl": 0.03341459959019716, "brier": 0.03316085045000001, "confidence": 0.512098, "qid": "needs_human" }, { "type": "score", "correct": false, "kl": 0.021853212672636298, "brier": 0.01541747168999999, "pred_score": 1.540049, "gold_score": 1.666667, "confidence": 0.46551, "qid": "urgency" } ] }, { "id": "customer_service_000022", "workflow": "customer_service", "decisions": [ { "type": "choice", "correct": false, "kl": 0.41288097100822396, "brier": 0.26669302942199996, "confidence": 0.544222, "qid": "action" }, { "type": "choice", "correct": true, "kl": 0.013587482559587928, "brier": 0.0008166127949999991, "confidence": 0.870966, "qid": "category" }, { "type": "score", "correct": false, "kl": 0.24871572473447828, "brier": 0.14494089716699998, "pred_score": 1.8777469999999998, "gold_score": 1.4, "confidence": 0.482647, "qid": "churn_risk" }, { "type": "noul", "correct": false, "kl": 0.2581055407242049, "brier": 0.249412894088, "confidence": 0.653138, "qid": "needs_human" }, { "type": "score", "correct": true, "kl": 0.08699957883409729, "brier": 0.05004098887799999, "pred_score": 2.773138, "gold_score": 2.556667, "confidence": 0.82414, "qid": "urgency" } ] }, { "id": "customer_service_000023", "workflow": "customer_service", "decisions": [ { "type": "choice", "correct": false, "kl": 0.7741135998320376, "brier": 0.31647226969700004, "confidence": 0.402223, "qid": "action" }, { "type": "choice", "correct": true, "kl": 0.13773409613218313, "brier": 0.03375706801599999, "confidence": 0.938917, "qid": "category" }, { "type": "score", "correct": true, "kl": 0.024796948485727242, "brier": 0.007800152451, "pred_score": 1.193889, "gold_score": 1.333333, "confidence": 0.525408, "qid": "churn_risk" }, { "type": "noul", "correct": false, "kl": 0.07486933598755027, "brier": 0.07377562368799997, "confidence": 0.525395, "qid": "needs_human" }, { "type": "score", "correct": false, "kl": 0.029644287294544713, "brier": 0.015695183294999995, "pred_score": 1.689563, "gold_score": 1.516667, "confidence": 0.429355, "qid": "urgency" } ] }, { "id": "customer_service_000024", "workflow": "customer_service", "decisions": [ { "type": "choice", "correct": true, "kl": 0.03547524775016532, "brier": 0.00410266419723632, "confidence": 0.49178049178049177, "qid": "action" }, { "type": "choice", "correct": true, "kl": 0.051533170038979274, "brier": 0.008791419994000002, "confidence": 0.897246, "qid": "category" }, { "type": "score", "correct": true, "kl": 0.0829410328028071, "brier": 0.009385851016000002, "pred_score": 1.669616, "gold_score": 1.883333, "confidence": 0.540906, "qid": "churn_risk" }, { "type": "noul", "correct": true, "kl": 0.0028579670145278263, "brier": 0.002737999999999992, "confidence": 0.589667, "qid": "needs_human" }, { "type": "score", "correct": true, "kl": 0.04286479514667935, "brier": 0.006853099602000001, "pred_score": 2.8264910000000003, "gold_score": 2.933333, "confidence": 0.864412, "qid": "urgency" } ] }, { "id": "customer_service_000025", "workflow": "customer_service", "decisions": [ { "type": "choice", "correct": true, "kl": 0.2150773028006912, "brier": 0.0964861047408661, "confidence": 0.4853765146234854, "qid": "action" }, { "type": "choice", "correct": true, "kl": 0.033426802050497195, "brier": 0.0017094868530000008, "confidence": 0.76217, "qid": "category" }, { "type": "score", "correct": true, "kl": 0.017115326237210557, "brier": 0.0075227434860000025, "pred_score": 2.068853, "gold_score": 2.01, "confidence": 0.641845, "qid": "churn_risk" }, { "type": "noul", "correct": true, "kl": 0.013362094226232343, "brier": 0.011526478112000003, "confidence": 0.709249, "qid": "needs_human" }, { "type": "score", "correct": true, "kl": 0.0027584076388661636, "brier": 0.002079755249999998, "pred_score": 1.6135929999999998, "gold_score": 1.583333, "confidence": 0.528118, "qid": "urgency" } ] }, { "id": "customer_service_000026", "workflow": "customer_service", "decisions": [ { "type": "choice", "correct": false, "kl": 0.2589486659813998, "brier": 0.153139897886, "confidence": 0.615269, "qid": "action" }, { "type": "choice", "correct": true, "kl": 0.04312637404271891, "brier": 0.004923387237999997, "confidence": 0.818945, "qid": "category" }, { "type": "score", "correct": true, "kl": 0.029866707280855276, "brier": 0.008502445520000013, "pred_score": 1.8970270000000002, "gold_score": 1.766667, "confidence": 0.6845330000000001, "qid": "churn_risk" }, { "type": "noul", "correct": false, "kl": 0.22920535053902283, "brier": 0.18910266012799998, "confidence": 0.790825, "qid": "needs_human" }, { "type": "score", "correct": true, "kl": 0.03370986610428791, "brier": 0.006515284555999997, "pred_score": 1.979368, "gold_score": 1.816667, "confidence": 0.598113, "qid": "urgency" } ] }, { "id": "customer_service_000027", "workflow": "customer_service", "decisions": [ { "type": "choice", "correct": true, "kl": 0.0387742465247346, "brier": 0.02019959861399998, "confidence": 0.754415, "qid": "action" }, { "type": "choice", "correct": true, "kl": 0.11976900076762595, "brier": 0.036241908520000005, "confidence": 0.649113, "qid": "category" }, { "type": "score", "correct": true, "kl": 0.011588767683188715, "brier": 0.0020697296130000047, "pred_score": 2.166221, "gold_score": 2.116667, "confidence": 0.635646, "qid": "churn_risk" }, { "type": "noul", "correct": true, "kl": 0.04835502640489295, "brier": 0.02812954805, "confidence": 0.858595, "qid": "needs_human" }, { "type": "score", "correct": true, "kl": 0.027119687976767268, "brier": 0.018064322520999977, "pred_score": 1.907853, "gold_score": 1.866667, "confidence": 0.654252, "qid": "urgency" } ] }, { "id": "customer_service_000028", "workflow": "customer_service", "decisions": [ { "type": "choice", "correct": true, "kl": 0.23173335167994513, "brier": 0.161798747971, "confidence": 0.436195, "qid": "action" }, { "type": "choice", "correct": true, "kl": 0.03925527683211773, "brier": 0.0014755184059999952, "confidence": 0.966505, "qid": "category" }, { "type": "score", "correct": false, "kl": 0.3183574046274314, "brier": 0.19657966464399995, "pred_score": 1.581504, "gold_score": 1.056667, "confidence": 0.464476, "qid": "churn_risk" }, { "type": "noul", "correct": false, "kl": 0.15840399821945794, "brier": 0.15455799216199997, "confidence": 0.554658, "qid": "needs_human" }, { "type": "score", "correct": false, "kl": 0.367626272367257, "brier": 0.2245085451574842, "pred_score": 1.921750078249922, "gold_score": 1.31, "confidence": 0.44892355107644893, "qid": "urgency" } ] }, { "id": "customer_service_000029", "workflow": "customer_service", "decisions": [ { "type": "choice", "correct": true, "kl": 0.18795222149207458, "brier": 0.038960183875000005, "confidence": 0.555323, "qid": "action" }, { "type": "choice", "correct": false, "kl": 1.3138565311623116, "brier": 0.6381744373110001, "confidence": 0.723873, "qid": "category" }, { "type": "score", "correct": true, "kl": 0.229339936237083, "brier": 0.12089584678200001, "pred_score": 2.266297, "gold_score": 2.83, "confidence": 0.580216, "qid": "churn_risk" }, { "type": "noul", "correct": true, "kl": 0.0011958657358644535, "brier": 0.0008707799120000008, "confidence": 0.767533, "qid": "needs_human" }, { "type": "score", "correct": true, "kl": 0.08064511695084466, "brier": 0.030783079642999995, "pred_score": 2.122374, "gold_score": 1.866667, "confidence": 0.586852, "qid": "urgency" } ] }, { "id": "customer_service_000030", "workflow": "customer_service", "decisions": [ { "type": "choice", "correct": true, "kl": 0.10141087468282683, "brier": 0.05631679648800001, "confidence": 0.536757, "qid": "action" }, { "type": "choice", "correct": true, "kl": 0.018678762108692397, "brier": 0.0028668900099999973, "confidence": 0.8121700000000001, "qid": "category" }, { "type": "score", "correct": true, "kl": 0.08544349536761209, "brier": 0.059089424656, "pred_score": 0.937085, "gold_score": 0.783333, "confidence": 0.661759, "qid": "churn_risk" }, { "type": "noul", "correct": true, "kl": 0.03981007671173099, "brier": 0.03630057345800002, "confidence": 0.59861, "qid": "needs_human" }, { "type": "score", "correct": true, "kl": 0.03669442927979448, "brier": 0.024558837785999998, "pred_score": 1.294975, "gold_score": 1.45, "confidence": 0.6756220000000001, "qid": "urgency" } ] }, { "id": "customer_service_000031", "workflow": "customer_service", "decisions": [ { "type": "choice", "correct": false, "kl": 0.3807707594209775, "brier": 0.243762965834, "confidence": 0.66713, "qid": "action" }, { "type": "choice", "correct": true, "kl": 0.04423539218205714, "brier": 0.00775000937699999, "confidence": 0.895634, "qid": "category" }, { "type": "score", "correct": true, "kl": 0.08889993202433669, "brier": 0.041641781909, "pred_score": 1.8480860000000001, "gold_score": 1.55, "confidence": 0.470515, "qid": "churn_risk" }, { "type": "noul", "correct": false, "kl": 0.2566322041023005, "brier": 0.22298242681800004, "confidence": 0.767236, "qid": "needs_human" }, { "type": "score", "correct": true, "kl": 0.0034084335010607695, "brier": 0.002217289567999999, "pred_score": 1.501312, "gold_score": 1.55, "confidence": 0.496507, "qid": "urgency" } ] }, { "id": "customer_service_000032", "workflow": "customer_service", "decisions": [ { "type": "choice", "correct": false, "kl": 0.26541350380810697, "brier": 0.154559516108, "confidence": 0.530166, "qid": "action" }, { "type": "choice", "correct": true, "kl": 0.0671576752079043, "brier": 0.01594513604399999, "confidence": 0.870445, "qid": "category" }, { "type": "score", "correct": true, "kl": 0.008478835279398387, "brier": 0.0021534984149999942, "pred_score": 1.882786, "gold_score": 1.833333, "confidence": 0.571405, "qid": "churn_risk" }, { "type": "noul", "correct": true, "kl": 0.04987467548467006, "brier": 0.044051649841999996, "confidence": 0.715078, "qid": "needs_human" }, { "type": "score", "correct": true, "kl": 0.010323182124400088, "brier": 0.00017483951000000017, "pred_score": 2.038185, "gold_score": 2.066667, "confidence": 0.695023, "qid": "urgency" } ] }, { "id": "customer_service_000033", "workflow": "customer_service", "decisions": [ { "type": "choice", "correct": false, "kl": 0.3331542780104172, "brier": 0.18873470598899997, "confidence": 0.637945, "qid": "action" }, { "type": "choice", "correct": false, "kl": 0.755560668576636, "brier": 0.4897904474460001, "confidence": 0.443234, "qid": "category" }, { "type": "score", "correct": true, "kl": 0.054294455184677806, "brier": 0.029405745654000018, "pred_score": 1.0739480000000001, "gold_score": 1.066667, "confidence": 0.669474, "qid": "churn_risk" }, { "type": "noul", "correct": true, "kl": 0.018999212176733477, "brier": 0.017509192711999993, "confidence": 0.606434, "qid": "needs_human" }, { "type": "score", "correct": false, "kl": 0.3460537228791068, "brier": 0.22383110954600002, "pred_score": 1.435185, "gold_score": 2.0, "confidence": 0.553581, "qid": "urgency" } ] }, { "id": "customer_service_000034", "workflow": "customer_service", "decisions": [ { "type": "choice", "correct": true, "kl": 0.03518515746524341, "brier": 0.018393925034999994, "confidence": 0.53677, "qid": "action" }, { "type": "choice", "correct": true, "kl": 0.04335061031171948, "brier": 0.005371964097999995, "confidence": 0.783643, "qid": "category" }, { "type": "score", "correct": true, "kl": 0.13512924599143555, "brier": 0.055040359806, "pred_score": 1.9438609999999998, "gold_score": 2.3, "confidence": 0.41764, "qid": "churn_risk" }, { "type": "noul", "correct": true, "kl": 0.009170954416729112, "brier": 0.006643355912000004, "confidence": 0.742366, "qid": "needs_human" }, { "type": "score", "correct": false, "kl": 0.21218728759777394, "brier": 0.16474931632899997, "pred_score": 2.302528, "gold_score": 2.7, "confidence": 0.476803, "qid": "urgency" } ] }, { "id": "customer_service_000035", "workflow": "customer_service", "decisions": [ { "type": "choice", "correct": false, "kl": 0.15991098789549044, "brier": 0.07704481005700001, "confidence": 0.515354, "qid": "action" }, { "type": "choice", "correct": true, "kl": 0.05037843852123071, "brier": 0.008947912147000008, "confidence": 0.907138, "qid": "category" }, { "type": "score", "correct": true, "kl": 0.16825196829601663, "brier": 0.057811490441999985, "pred_score": 1.969809, "gold_score": 1.566667, "confidence": 0.508052, "qid": "churn_risk" }, { "type": "noul", "correct": true, "kl": 0.007752470759716239, "brier": 0.006432275042000009, "confidence": 0.686622, "qid": "needs_human" }, { "type": "score", "correct": true, "kl": 0.0149963268715574, "brier": 0.004298499794000003, "pred_score": 2.308368, "gold_score": 2.2, "confidence": 0.459228, "qid": "urgency" } ] }, { "id": "customer_service_000036", "workflow": "customer_service", "decisions": [ { "type": "choice", "correct": true, "kl": 0.10627390995577829, "brier": 0.01558196572399999, "confidence": 0.722798, "qid": "action" }, { "type": "choice", "correct": true, "kl": 0.1663973208016828, "brier": 0.06585843687400002, "confidence": 0.691009, "qid": "category" }, { "type": "score", "correct": true, "kl": 0.1359463771462731, "brier": 0.026993251214, "pred_score": 2.519601, "gold_score": 2.866667, "confidence": 0.761209, "qid": "churn_risk" }, { "type": "noul", "correct": true, "kl": 0.03783706108819991, "brier": 0.025520375042000006, "confidence": 0.743706, "qid": "needs_human" }, { "type": "score", "correct": false, "kl": 1.2089915949694472, "brier": 0.545790945602, "pred_score": 1.215516, "gold_score": 2.333333, "confidence": 0.666771, "qid": "urgency" } ] }, { "id": "customer_service_000037", "workflow": "customer_service", "decisions": [ { "type": "choice", "correct": true, "kl": 0.13074034833105685, "brier": 0.02260289457, "confidence": 0.622106, "qid": "action" }, { "type": "choice", "correct": true, "kl": 0.10620086865551882, "brier": 0.015091137033999991, "confidence": 0.899244, "qid": "category" }, { "type": "score", "correct": true, "kl": 0.008776723796734703, "brier": 0.002157984010999998, "pred_score": 1.771503, "gold_score": 1.85, "confidence": 0.639847, "qid": "churn_risk" }, { "type": "noul", "correct": true, "kl": 0.01537260184374184, "brier": 0.010449953312, "confidence": 0.805617, "qid": "needs_human" }, { "type": "score", "correct": false, "kl": 0.058624020410571284, "brier": 0.045017010276000004, "pred_score": 1.678681, "gold_score": 1.533333, "confidence": 0.585318, "qid": "urgency" } ] }, { "id": "customer_service_000038", "workflow": "customer_service", "decisions": [ { "type": "choice", "correct": true, "kl": 0.19346004371000342, "brier": 0.136017667176, "confidence": 0.367864, "qid": "action" }, { "type": "choice", "correct": true, "kl": 0.024094366930139506, "brier": 0.005361825276999992, "confidence": 0.859701, "qid": "category" }, { "type": "score", "correct": true, "kl": 0.1395710577098758, "brier": 0.051363948266000006, "pred_score": 1.147064, "gold_score": 0.833333, "confidence": 0.596206, "qid": "churn_risk" }, { "type": "noul", "correct": true, "kl": 0.08347173803581129, "brier": 0.074403845768, "confidence": 0.590455, "qid": "needs_human" }, { "type": "score", "correct": true, "kl": 0.09611321491709282, "brier": 0.028463673168000008, "pred_score": 1.491902, "gold_score": 1.383333, "confidence": 0.534757, "qid": "urgency" } ] }, { "id": "customer_service_000039", "workflow": "customer_service", "decisions": [ { "type": "choice", "correct": true, "kl": 0.11049770102176301, "brier": 0.029896865747999997, "confidence": 0.627373, "qid": "action" }, { "type": "choice", "correct": true, "kl": 0.029749319794753207, "brier": 0.0034469150409999955, "confidence": 0.929312, "qid": "category" }, { "type": "score", "correct": true, "kl": 0.15228873066048612, "brier": 0.041884924058999996, "pred_score": 0.88474, "gold_score": 1.266667, "confidence": 0.579572, "qid": "churn_risk" }, { "type": "noul", "correct": true, "kl": 0.01685237741631536, "brier": 0.015111911250000002, "confidence": 0.629742, "qid": "needs_human" }, { "type": "score", "correct": true, "kl": 0.030810848101430033, "brier": 0.018926966373999994, "pred_score": 1.1789830000000001, "gold_score": 1.216667, "confidence": 0.682759, "qid": "urgency" } ] }, { "id": "customer_service_000040", "workflow": "customer_service", "decisions": [ { "type": "choice", "correct": true, "kl": 0.2318682508164545, "brier": 0.127804620777, "confidence": 0.690215, "qid": "action" }, { "type": "choice", "correct": true, "kl": 0.1046672022510155, "brier": 0.02688409386599999, "confidence": 0.808134, "qid": "category" }, { "type": "score", "correct": true, "kl": 0.041493038821714144, "brier": 0.009265122901999994, "pred_score": 1.9986629999999999, "gold_score": 2.166667, "confidence": 0.584178, "qid": "churn_risk" }, { "type": "noul", "correct": true, "kl": 0.005793173323091108, "brier": 0.004542379298000006, "confidence": 0.747657, "qid": "needs_human" }, { "type": "score", "correct": true, "kl": 0.00678259579440061, "brier": 0.0005858889900000019, "pred_score": 2.85011, "gold_score": 2.866667, "confidence": 0.885006, "qid": "urgency" } ] }, { "id": "customer_service_000041", "workflow": "customer_service", "decisions": [ { "type": "choice", "correct": true, "kl": 0.06575071920836147, "brier": 0.006590861781999997, "confidence": 0.7528, "qid": "action" }, { "type": "choice", "correct": true, "kl": 0.04237333989801644, "brier": 0.005290226619999997, "confidence": 0.854944, "qid": "category" }, { "type": "score", "correct": true, "kl": 0.05627810667508726, "brier": 0.009004823918000004, "pred_score": 2.678516, "gold_score": 2.516667, "confidence": 0.73184, "qid": "churn_risk" }, { "type": "noul", "correct": true, "kl": 0.0017447334888561032, "brier": 0.0008235305279999992, "confidence": 0.856375, "qid": "needs_human" }, { "type": "score", "correct": true, "kl": 0.07044686125961974, "brier": 0.021330548670000005, "pred_score": 2.6300540000000003, "gold_score": 2.41, "confidence": 0.6916800000000001, "qid": "urgency" } ] }, { "id": "customer_service_000042", "workflow": "customer_service", "decisions": [ { "type": "choice", "correct": true, "kl": 0.14598677908502067, "brier": 0.09075662647999999, "confidence": 0.442546, "qid": "action" }, { "type": "choice", "correct": true, "kl": 0.12371065293933378, "brier": 0.017203051173999992, "confidence": 0.853895, "qid": "category" }, { "type": "score", "correct": true, "kl": 0.01148947388491267, "brier": 0.0011006909020000002, "pred_score": 2.346715, "gold_score": 2.366667, "confidence": 0.4927, "qid": "churn_risk" }, { "type": "noul", "correct": true, "kl": 0.031301605767362295, "brier": 0.023637032738000007, "confidence": 0.707954, "qid": "needs_human" }, { "type": "score", "correct": true, "kl": 0.008451565301142227, "brier": 0.0003211189650189998, "pred_score": 2.3887716112283885, "gold_score": 2.416667, "confidence": 0.4862415137584862, "qid": "urgency" } ] }, { "id": "customer_service_000043", "workflow": "customer_service", "decisions": [ { "type": "choice", "correct": false, "kl": 0.6139768682204347, "brier": 0.279162412227, "confidence": 0.628424, "qid": "action" }, { "type": "choice", "correct": true, "kl": 0.023441332100509055, "brier": 0.0021780094410000003, "confidence": 0.780361, "qid": "category" }, { "type": "score", "correct": true, "kl": 0.03706692947706483, "brier": 0.008660453914000001, "pred_score": 1.9654630000000002, "gold_score": 2.1, "confidence": 0.64195, "qid": "churn_risk" }, { "type": "noul", "correct": true, "kl": 0.00033724555383025413, "brier": 0.0002685634880000007, "confidence": 0.721745, "qid": "needs_human" }, { "type": "score", "correct": true, "kl": 0.022994115066094938, "brier": 0.01660499976599999, "pred_score": 2.020734, "gold_score": 1.983333, "confidence": 0.496852, "qid": "urgency" } ] }, { "id": "customer_service_000044", "workflow": "customer_service", "decisions": [ { "type": "choice", "correct": true, "kl": 0.08888026717681394, "brier": 0.03457671853651341, "confidence": 0.4206664206664207, "qid": "action" }, { "type": "choice", "correct": true, "kl": 0.1091045886854425, "brier": 0.04688035053200001, "confidence": 0.735309, "qid": "category" }, { "type": "score", "correct": false, "kl": 0.19044218329677484, "brier": 0.07723200403800001, "pred_score": 1.160526, "gold_score": 1.583333, "confidence": 0.555485, "qid": "churn_risk" }, { "type": "noul", "correct": true, "kl": 0.12250963576599688, "brier": 0.11014189516799999, "confidence": 0.565328, "qid": "needs_human" }, { "type": "score", "correct": true, "kl": 0.0507652394208429, "brier": 0.013096235737999997, "pred_score": 2.062586, "gold_score": 2.233333, "confidence": 0.59714, "qid": "urgency" } ] }, { "id": "customer_service_000045", "workflow": "customer_service", "decisions": [ { "type": "choice", "correct": true, "kl": 0.06885360178547902, "brier": 0.019754682654000003, "confidence": 0.78345, "qid": "action" }, { "type": "choice", "correct": true, "kl": 0.006288126796701104, "brier": 8.735086399999997e-05, "confidence": 0.974518, "qid": "category" }, { "type": "score", "correct": true, "kl": 0.1800639447006148, "brier": 0.11027388151399999, "pred_score": 0.44281200000000004, "gold_score": 0.116667, "confidence": 0.636688, "qid": "churn_risk" }, { "type": "noul", "correct": true, "kl": 0.026842312307639657, "brier": 0.012051902258, "confidence": 0.842373, "qid": "needs_human" }, { "type": "score", "correct": true, "kl": 0.0722998468576706, "brier": 0.03536005087799998, "pred_score": 2.569263, "gold_score": 2.813333, "confidence": 0.693157, "qid": "urgency" } ] }, { "id": "customer_service_000046", "workflow": "customer_service", "decisions": [ { "type": "choice", "correct": true, "kl": 0.3664466817289782, "brier": 0.119814012152, "confidence": 0.473558, "qid": "action" }, { "type": "choice", "correct": true, "kl": 0.039955276537875006, "brier": 0.0023673608360000016, "confidence": 0.935992, "qid": "category" }, { "type": "score", "correct": false, "kl": 0.06299962523966836, "brier": 0.050622329224000014, "pred_score": 1.541228, "gold_score": 1.383333, "confidence": 0.427705, "qid": "churn_risk" }, { "type": "noul", "correct": true, "kl": 0.014668735505407565, "brier": 0.01339688067199999, "confidence": 0.618156, "qid": "needs_human" }, { "type": "score", "correct": true, "kl": 0.02104356938317679, "brier": 0.004496235229000001, "pred_score": 2.148183, "gold_score": 2.25, "confidence": 0.561005, "qid": "urgency" } ] }, { "id": "customer_service_000047", "workflow": "customer_service", "decisions": [ { "type": "choice", "correct": true, "kl": 0.11568080990372445, "brier": 0.070784238896, "confidence": 0.440226, "qid": "action" }, { "type": "choice", "correct": true, "kl": 0.054630516401179, "brier": 0.02194850844512517, "confidence": 0.4763774763774764, "qid": "category" }, { "type": "score", "correct": false, "kl": 0.032194827503570084, "brier": 0.014223967797000008, "pred_score": 1.774446, "gold_score": 1.566667, "confidence": 0.433527, "qid": "churn_risk" }, { "type": "noul", "correct": false, "kl": 0.09014649900422456, "brier": 0.08786264320799998, "confidence": 0.509598, "qid": "needs_human" }, { "type": "score", "correct": true, "kl": 0.02784366022387199, "brier": 0.006058357457000005, "pred_score": 2.439406, "gold_score": 2.4, "confidence": 0.566481, "qid": "urgency" } ] }, { "id": "customer_service_000048", "workflow": "customer_service", "decisions": [ { "type": "choice", "correct": true, "kl": 0.12716457324841307, "brier": 0.063989028582, "confidence": 0.558716, "qid": "action" }, { "type": "choice", "correct": true, "kl": 0.02955249561930005, "brier": 0.0029329915449999963, "confidence": 0.940303, "qid": "category" }, { "type": "score", "correct": true, "kl": 0.04790234746305074, "brier": 0.013481590766000005, "pred_score": 1.897593, "gold_score": 2.033333, "confidence": 0.57802, "qid": "churn_risk" }, { "type": "noul", "correct": true, "kl": 0.025506337061570572, "brier": 0.019810451250000014, "confidence": 0.700475, "qid": "needs_human" }, { "type": "score", "correct": true, "kl": 0.043735678648282164, "brier": 0.011034323581999997, "pred_score": 1.9954460000000003, "gold_score": 2.166667, "confidence": 0.613867, "qid": "urgency" } ] }, { "id": "customer_service_000049", "workflow": "customer_service", "decisions": [ { "type": "choice", "correct": false, "kl": 0.34383497170857186, "brier": 0.19271221058199997, "confidence": 0.554009, "qid": "action" }, { "type": "choice", "correct": true, "kl": 0.06521188325860099, "brier": 0.006920138956000006, "confidence": 0.818978, "qid": "category" }, { "type": "score", "correct": true, "kl": 0.08198494031210524, "brier": 0.027234645039000006, "pred_score": 1.783012, "gold_score": 2.016667, "confidence": 0.607796, "qid": "churn_risk" }, { "type": "noul", "correct": false, "kl": 0.0727713073181547, "brier": 0.06879453244999999, "confidence": 0.668798, "qid": "needs_human" }, { "type": "score", "correct": true, "kl": 0.1020745016757093, "brier": 0.040141585098999996, "pred_score": 2.821241, "gold_score": 2.7, "confidence": 0.870745, "qid": "urgency" } ] }, { "id": "customer_service_000050", "workflow": "customer_service", "decisions": [ { "type": "choice", "correct": true, "kl": 0.010339538707391922, "brier": 0.0012764375550000002, "confidence": 0.711491, "qid": "action" }, { "type": "choice", "correct": true, "kl": 0.02104900268851201, "brier": 0.004899560089999998, "confidence": 0.67737, "qid": "category" }, { "type": "score", "correct": false, "kl": 0.02496914075848783, "brier": 0.018745770872999987, "pred_score": 2.422664, "gold_score": 2.3, "confidence": 0.488235, "qid": "churn_risk" }, { "type": "noul", "correct": true, "kl": 0.0001402508510646456, "brier": 8.24584819999999e-05, "confidence": 0.823088, "qid": "needs_human" }, { "type": "score", "correct": false, "kl": 0.027958858509168967, "brier": 0.015448632859273156, "pred_score": 2.3220996779003222, "gold_score": 2.4, "confidence": 0.45869854130145876, "qid": "urgency" } ] }, { "id": "customer_service_000051", "workflow": "customer_service", "decisions": [ { "type": "choice", "correct": true, "kl": 0.03622246298031436, "brier": 0.011881093096000014, "confidence": 0.719127, "qid": "action" }, { "type": "choice", "correct": true, "kl": 0.08593093986934566, "brier": 0.05920822069099999, "confidence": 0.570713, "qid": "category" }, { "type": "score", "correct": true, "kl": 0.0829286582780073, "brier": 0.059189411873999995, "pred_score": 2.572185, "gold_score": 2.783333, "confidence": 0.633514, "qid": "churn_risk" }, { "type": "noul", "correct": true, "kl": 0.0037624091436281627, "brier": 0.001769649032000002, "confidence": 0.853587, "qid": "needs_human" }, { "type": "score", "correct": true, "kl": 0.0070634120759176, "brier": 0.0010424819790000013, "pred_score": 2.7535849999999997, "gold_score": 2.75, "confidence": 0.800593, "qid": "urgency" } ] }, { "id": "customer_service_000052", "workflow": "customer_service", "decisions": [ { "type": "choice", "correct": true, "kl": 0.4021639585239069, "brier": 0.14737766039, "confidence": 0.503326, "qid": "action" }, { "type": "choice", "correct": true, "kl": 0.008085611480164451, "brier": 0.004553176893000002, "confidence": 0.622843, "qid": "category" }, { "type": "score", "correct": true, "kl": 0.07431703853402372, "brier": 0.05295737345400002, "pred_score": 2.5956029999999997, "gold_score": 2.783333, "confidence": 0.647107, "qid": "churn_risk" }, { "type": "noul", "correct": true, "kl": 0.07871508626979407, "brier": 0.06015315495199999, "confidence": 0.676574, "qid": "needs_human" }, { "type": "score", "correct": true, "kl": 0.01312543741678844, "brier": 0.0035153931620000058, "pred_score": 2.8654360000000003, "gold_score": 2.833333, "confidence": 0.889449, "qid": "urgency" } ] }, { "id": "customer_service_000053", "workflow": "customer_service", "decisions": [ { "type": "choice", "correct": false, "kl": 0.19297663906007875, "brier": 0.13061758090626815, "confidence": 0.4534714534714535, "qid": "action" }, { "type": "choice", "correct": true, "kl": 0.06076518042405832, "brier": 0.015258241390000003, "confidence": 0.848212, "qid": "category" }, { "type": "score", "correct": true, "kl": 0.017403531958683098, "brier": 0.008378682065999999, "pred_score": 1.971807, "gold_score": 1.933333, "confidence": 0.51732, "qid": "churn_risk" }, { "type": "noul", "correct": true, "kl": 0.0012408978951402756, "brier": 0.0012064854419999971, "confidence": 0.591228, "qid": "needs_human" }, { "type": "score", "correct": true, "kl": 0.003713090501688762, "brier": 0.002855087965999996, "pred_score": 2.360909, "gold_score": 2.316667, "confidence": 0.522264, "qid": "urgency" } ] }, { "id": "customer_service_000054", "workflow": "customer_service", "decisions": [ { "type": "choice", "correct": true, "kl": 0.12425964313563337, "brier": 0.055679591906, "confidence": 0.705566, "qid": "action" }, { "type": "choice", "correct": true, "kl": 0.2757072107363545, "brier": 0.19591009643336552, "confidence": 0.47929347929347926, "qid": "category" }, { "type": "score", "correct": true, "kl": 0.20694497601148054, "brier": 0.10656310275599996, "pred_score": 2.614585, "gold_score": 2.933333, "confidence": 0.678046, "qid": "churn_risk" }, { "type": "noul", "correct": true, "kl": 0.11033277713430982, "brier": 0.05426140359200002, "confidence": 0.788619, "qid": "needs_human" }, { "type": "score", "correct": true, "kl": 0.021630562900849575, "brier": 0.000633030165999999, "pred_score": 2.879395, "gold_score": 2.916667, "confidence": 0.902383, "qid": "urgency" } ] }, { "id": "customer_service_000055", "workflow": "customer_service", "decisions": [ { "type": "choice", "correct": false, "kl": 0.4454899529324395, "brier": 0.313171450294, "confidence": 0.528891, "qid": "action" }, { "type": "choice", "correct": true, "kl": 0.26082323539099384, "brier": 0.14668952104799998, "confidence": 0.77792, "qid": "category" }, { "type": "score", "correct": false, "kl": 0.15550863229813636, "brier": 0.12740325142600004, "pred_score": 2.427773, "gold_score": 2.166667, "confidence": 0.5248, "qid": "churn_risk" }, { "type": "noul", "correct": true, "kl": 0.1356360213557991, "brier": 0.11295314380800003, "confidence": 0.770981, "qid": "needs_human" }, { "type": "score", "correct": true, "kl": 0.04012248167592388, "brier": 0.014954368807, "pred_score": 1.959416, "gold_score": 1.9, "confidence": 0.723044, "qid": "urgency" } ] }, { "id": "customer_service_000056", "workflow": "customer_service", "decisions": [ { "type": "choice", "correct": true, "kl": 0.42621885916186775, "brier": 0.118692522685, "confidence": 0.632679, "qid": "action" }, { "type": "choice", "correct": true, "kl": 0.04004191937053962, "brier": 0.004580514934000006, "confidence": 0.891311, "qid": "category" }, { "type": "score", "correct": true, "kl": 0.12481445962760646, "brier": 0.068189789364, "pred_score": 2.8020470000000004, "gold_score": 2.636667, "confidence": 0.839381, "qid": "churn_risk" }, { "type": "noul", "correct": true, "kl": 0.00024017511383527153, "brier": 0.00023358249800000048, "confidence": 0.579193, "qid": "needs_human" }, { "type": "score", "correct": true, "kl": 0.06942922886143377, "brier": 0.04966076598999998, "pred_score": 2.6240989999999997, "gold_score": 2.403333, "confidence": 0.689448, "qid": "urgency" } ] }, { "id": "customer_service_000057", "workflow": "customer_service", "decisions": [ { "type": "choice", "correct": true, "kl": 0.09971021700321678, "brier": 0.010442092847000001, "confidence": 0.79289, "qid": "action" }, { "type": "choice", "correct": true, "kl": 0.0645291626525585, "brier": 0.007546234234999994, "confidence": 0.925964, "qid": "category" }, { "type": "score", "correct": true, "kl": 0.05465853548252411, "brier": 0.021255213135, "pred_score": 2.668293, "gold_score": 2.85, "confidence": 0.745856, "qid": "churn_risk" }, { "type": "noul", "correct": true, "kl": 0.12516081421712114, "brier": 0.109230445202, "confidence": 0.743699, "qid": "needs_human" }, { "type": "score", "correct": true, "kl": 0.023200864276934553, "brier": 0.0006491701340000017, "pred_score": 2.848417, "gold_score": 2.883333, "confidence": 0.873574, "qid": "urgency" } ] }, { "id": "customer_service_000058", "workflow": "customer_service", "decisions": [ { "type": "choice", "correct": false, "kl": 0.26889114446848283, "brier": 0.21497244124000003, "confidence": 0.594677, "qid": "action" }, { "type": "choice", "correct": true, "kl": 0.06354553232983103, "brier": 0.007685698223000009, "confidence": 0.917314, "qid": "category" }, { "type": "score", "correct": true, "kl": 0.0611896165337668, "brier": 0.025662676681000002, "pred_score": 2.5995649999999997, "gold_score": 2.8, "confidence": 0.684242, "qid": "churn_risk" }, { "type": "noul", "correct": true, "kl": 0.0748291349032481, "brier": 0.059484213362, "confidence": 0.660874, "qid": "needs_human" }, { "type": "score", "correct": true, "kl": 0.09046753589435554, "brier": 0.05868182778599999, "pred_score": 2.200945, "gold_score": 2.15, "confidence": 0.553366, "qid": "urgency" } ] }, { "id": "customer_service_000059", "workflow": "customer_service", "decisions": [ { "type": "choice", "correct": true, "kl": 0.2281659646600438, "brier": 0.139926759002, "confidence": 0.655228, "qid": "action" }, { "type": "choice", "correct": true, "kl": 0.06387992126014899, "brier": 0.009815595715999997, "confidence": 0.5400510000000001, "qid": "category" }, { "type": "score", "correct": true, "kl": 0.016876130065069134, "brier": 0.0035673252481134648, "pred_score": 2.2840692840692842, "gold_score": 2.233333, "confidence": 0.498001498001498, "qid": "churn_risk" }, { "type": "noul", "correct": true, "kl": 0.062327121016431675, "brier": 0.04842832176200001, "confidence": 0.782276, "qid": "needs_human" }, { "type": "score", "correct": true, "kl": 0.02488244767856515, "brier": 0.010726751711999985, "pred_score": 2.595938, "gold_score": 2.483333, "confidence": 0.649631, "qid": "urgency" } ] }, { "id": "customer_service_000060", "workflow": "customer_service", "decisions": [ { "type": "choice", "correct": false, "kl": 0.29463644022350854, "brier": 0.12244879310600001, "confidence": 0.411799, "qid": "action" }, { "type": "choice", "correct": true, "kl": 0.018064515243954724, "brier": 0.0017385497229999985, "confidence": 0.943388, "qid": "category" }, { "type": "score", "correct": true, "kl": 0.23339758060642715, "brier": 0.12788093841278764, "pred_score": 1.6901546901546902, "gold_score": 2.066667, "confidence": 0.39943339943339945, "qid": "churn_risk" }, { "type": "noul", "correct": true, "kl": 0.00395333652820954, "brier": 0.003937748767999999, "confidence": 0.544372, "qid": "needs_human" }, { "type": "score", "correct": true, "kl": 0.011301978489522753, "brier": 0.0025834573350000008, "pred_score": 2.840265, "gold_score": 2.9, "confidence": 0.875112, "qid": "urgency" } ] }, { "id": "customer_service_000061", "workflow": "customer_service", "decisions": [ { "type": "choice", "correct": false, "kl": 0.23113142460982034, "brier": 0.14022526932799997, "confidence": 0.470633, "qid": "action" }, { "type": "choice", "correct": true, "kl": 0.0647622422331635, "brier": 0.014683150544000005, "confidence": 0.858009, "qid": "category" }, { "type": "score", "correct": true, "kl": 0.029688233025308902, "brier": 0.023762996154, "pred_score": 2.438143, "gold_score": 2.543333, "confidence": 0.543255, "qid": "churn_risk" }, { "type": "noul", "correct": false, "kl": 0.037722433444284664, "brier": 0.03673784604799999, "confidence": 0.618865, "qid": "needs_human" }, { "type": "score", "correct": true, "kl": 0.02108172687771276, "brier": 0.005363160939000004, "pred_score": 1.945642, "gold_score": 2.016667, "confidence": 0.732117, "qid": "urgency" } ] }, { "id": "customer_service_000062", "workflow": "customer_service", "decisions": [ { "type": "choice", "correct": true, "kl": 0.19770995210259706, "brier": 0.06513499569399998, "confidence": 0.343515, "qid": "action" }, { "type": "choice", "correct": true, "kl": 0.006517276685511062, "brier": 4.916379799999917e-05, "confidence": 0.970967, "qid": "category" }, { "type": "score", "correct": true, "kl": 0.058330005379677294, "brier": 0.027353708141999993, "pred_score": 1.1588209999999999, "gold_score": 1.033333, "confidence": 0.566912, "qid": "churn_risk" }, { "type": "noul", "correct": false, "kl": 0.03474821729842681, "brier": 0.03461133120199999, "confidence": 0.548218, "qid": "needs_human" }, { "type": "score", "correct": true, "kl": 0.050097048799351215, "brier": 0.010077276884999988, "pred_score": 1.481713, "gold_score": 1.4, "confidence": 0.530707, "qid": "urgency" } ] }, { "id": "customer_service_000063", "workflow": "customer_service", "decisions": [ { "type": "choice", "correct": true, "kl": 0.21447792539219365, "brier": 0.07685084836589984, "confidence": 0.4321125678874322, "qid": "action" }, { "type": "choice", "correct": true, "kl": 0.024482870499835274, "brier": 0.0009089299740000026, "confidence": 0.936911, "qid": "category" }, { "type": "score", "correct": true, "kl": 0.1082104530663137, "brier": 0.026522353933, "pred_score": 0.934017, "gold_score": 1.243333, "confidence": 0.567322, "qid": "churn_risk" }, { "type": "noul", "correct": true, "kl": 0.11599689609129507, "brier": 0.08967104307199997, "confidence": 0.654923, "qid": "needs_human" }, { "type": "score", "correct": true, "kl": 0.015362978865252175, "brier": 0.005695945997999997, "pred_score": 1.172735, "gold_score": 1.136667, "confidence": 0.711095, "qid": "urgency" } ] }, { "id": "customer_service_000064", "workflow": "customer_service", "decisions": [ { "type": "choice", "correct": true, "kl": 0.14449503728084148, "brier": 0.07595845759900001, "confidence": 0.640948, "qid": "action" }, { "type": "choice", "correct": true, "kl": 0.07455465179373046, "brier": 0.011277129026000001, "confidence": 0.89188, "qid": "category" }, { "type": "score", "correct": true, "kl": 0.13855840487681614, "brier": 0.05511454377000001, "pred_score": 1.51999, "gold_score": 1.233333, "confidence": 0.443761, "qid": "churn_risk" }, { "type": "noul", "correct": true, "kl": 0.08536536670189088, "brier": 0.050338375808000005, "confidence": 0.758019, "qid": "needs_human" }, { "type": "score", "correct": true, "kl": 0.01815621728825826, "brier": 0.010304668698000006, "pred_score": 2.212008, "gold_score": 2.166667, "confidence": 0.588842, "qid": "urgency" } ] }, { "id": "customer_service_000065", "workflow": "customer_service", "decisions": [ { "type": "choice", "correct": true, "kl": 0.024929909148183462, "brier": 0.003625174026000002, "confidence": 0.55251, "qid": "action" }, { "type": "choice", "correct": true, "kl": 0.04529867927883928, "brier": 0.030691987091000024, "confidence": 0.6544490000000001, "qid": "category" }, { "type": "score", "correct": true, "kl": 0.0075232316757307925, "brier": 0.002104921193999998, "pred_score": 2.726722, "gold_score": 2.766667, "confidence": 0.783043, "qid": "churn_risk" }, { "type": "noul", "correct": true, "kl": 0.0001231046100530148, "brier": 8.924480000000015e-05, "confidence": 0.759987, "qid": "needs_human" }, { "type": "score", "correct": true, "kl": 0.007389430891156779, "brier": 0.0012404930550000007, "pred_score": 2.400918, "gold_score": 2.4, "confidence": 0.484678, "qid": "urgency" } ] }, { "id": "customer_service_000066", "workflow": "customer_service", "decisions": [ { "type": "choice", "correct": true, "kl": 0.07381303606675454, "brier": 0.01862249229958981, "confidence": 0.4654275345724655, "qid": "action" }, { "type": "choice", "correct": true, "kl": 0.027789172229239597, "brier": 0.015839021215000006, "confidence": 0.752604, "qid": "category" }, { "type": "score", "correct": true, "kl": 0.031550277580801776, "brier": 0.008053230423, "pred_score": 1.8733710000000001, "gold_score": 1.7, "confidence": 0.536836, "qid": "churn_risk" }, { "type": "noul", "correct": true, "kl": 0.03032735582868981, "brier": 0.029425924417999995, "confidence": 0.621297, "qid": "needs_human" }, { "type": "score", "correct": true, "kl": 0.009618934190251859, "brier": 0.0008953513099999983, "pred_score": 2.387066, "gold_score": 2.366667, "confidence": 0.520523, "qid": "urgency" } ] }, { "id": "customer_service_000067", "workflow": "customer_service", "decisions": [ { "type": "choice", "correct": false, "kl": 0.2637022405469608, "brier": 0.16074683406955995, "confidence": 0.47900752099247906, "qid": "action" }, { "type": "choice", "correct": true, "kl": 0.018421993314339736, "brier": 0.0006266217840000004, "confidence": 0.915243, "qid": "category" }, { "type": "score", "correct": false, "kl": 0.11865051489465062, "brier": 0.093952159106, "pred_score": 1.5385849999999999, "gold_score": 1.25, "confidence": 0.491952, "qid": "churn_risk" }, { "type": "noul", "correct": true, "kl": 0.04869049016404464, "brier": 0.04639484449799999, "confidence": 0.652307, "qid": "needs_human" }, { "type": "score", "correct": true, "kl": 0.012682096269939224, "brier": 0.003078025276999998, "pred_score": 1.6621350000000001, "gold_score": 1.633333, "confidence": 0.48164, "qid": "urgency" } ] }, { "id": "customer_service_000068", "workflow": "customer_service", "decisions": [ { "type": "choice", "correct": true, "kl": 0.13381802620433522, "brier": 0.07653941462099999, "confidence": 0.530243, "qid": "action" }, { "type": "choice", "correct": true, "kl": 0.009686939016675349, "brier": 0.0001720350659999996, "confidence": 0.978551, "qid": "category" }, { "type": "score", "correct": true, "kl": 0.04613733741806364, "brier": 0.01316841678600001, "pred_score": 1.456598, "gold_score": 1.65, "confidence": 0.468636, "qid": "churn_risk" }, { "type": "noul", "correct": true, "kl": 0.01898226483862596, "brier": 0.016667363042000012, "confidence": 0.642044, "qid": "needs_human" }, { "type": "score", "correct": true, "kl": 0.013040342571580658, "brier": 0.0034576427179999997, "pred_score": 1.60742, "gold_score": 1.57, "confidence": 0.530795, "qid": "urgency" } ] }, { "id": "customer_service_000069", "workflow": "customer_service", "decisions": [ { "type": "choice", "correct": true, "kl": 0.01969771690627701, "brier": 0.011526542678999992, "confidence": 0.536725, "qid": "action" }, { "type": "choice", "correct": true, "kl": 0.059621116267538736, "brier": 0.005573235005, "confidence": 0.909993, "qid": "category" }, { "type": "score", "correct": true, "kl": 0.02361608741216427, "brier": 0.0033122897059999993, "pred_score": 2.000648, "gold_score": 1.966667, "confidence": 0.60696, "qid": "churn_risk" }, { "type": "noul", "correct": true, "kl": 0.026770534430200582, "brier": 0.021745299967999978, "confidence": 0.679061, "qid": "needs_human" }, { "type": "score", "correct": true, "kl": 0.012588612859827222, "brier": 0.007404247822002048, "pred_score": 2.257700257700258, "gold_score": 2.25, "confidence": 0.431015431015431, "qid": "urgency" } ] }, { "id": "customer_service_000070", "workflow": "customer_service", "decisions": [ { "type": "choice", "correct": true, "kl": 0.20091332467709164, "brier": 0.11059407790899999, "confidence": 0.395988, "qid": "action" }, { "type": "choice", "correct": true, "kl": 0.0415994529735106, "brier": 0.004751147888999995, "confidence": 0.933288, "qid": "category" }, { "type": "score", "correct": true, "kl": 0.016309291473216313, "brier": 0.006611393298999999, "pred_score": 1.258886, "gold_score": 1.266667, "confidence": 0.534728, "qid": "churn_risk" }, { "type": "noul", "correct": true, "kl": 0.11396525352571774, "brier": 0.10815993520199998, "confidence": 0.517449, "qid": "needs_human" }, { "type": "score", "correct": true, "kl": 0.057603871360844744, "brier": 0.014646995453000008, "pred_score": 1.5443639999999998, "gold_score": 1.316667, "confidence": 0.476226, "qid": "urgency" } ] }, { "id": "customer_service_000071", "workflow": "customer_service", "decisions": [ { "type": "choice", "correct": false, "kl": 0.3545985816363613, "brier": 0.22790248332004884, "confidence": 0.4106695893304107, "qid": "action" }, { "type": "choice", "correct": true, "kl": 0.03487878170178516, "brier": 0.005979161287000003, "confidence": 0.841042, "qid": "category" }, { "type": "score", "correct": true, "kl": 0.010622844129981605, "brier": 0.0053994153249999986, "pred_score": 1.347331, "gold_score": 1.266667, "confidence": 0.564276, "qid": "churn_risk" }, { "type": "noul", "correct": false, "kl": 0.2895073902990867, "brier": 0.2730442422420001, "confidence": 0.552822, "qid": "needs_human" }, { "type": "score", "correct": true, "kl": 0.013411968954318586, "brier": 0.007527205059105136, "pred_score": 1.8896621103378899, "gold_score": 2.006667, "confidence": 0.4138765861234139, "qid": "urgency" } ] }, { "id": "customer_service_000072", "workflow": "customer_service", "decisions": [ { "type": "choice", "correct": true, "kl": 0.2250226161856676, "brier": 0.06944351338, "confidence": 0.624934, "qid": "action" }, { "type": "choice", "correct": true, "kl": 0.09659767674775412, "brier": 0.024070688355999983, "confidence": 0.847542, "qid": "category" }, { "type": "score", "correct": true, "kl": 0.03532398596413805, "brier": 0.013869762977999986, "pred_score": 2.3757460000000004, "gold_score": 2.543333, "confidence": 0.5004890000000001, "qid": "churn_risk" }, { "type": "noul", "correct": true, "kl": 0.004795602254355345, "brier": 0.003607911457999997, "confidence": 0.734194, "qid": "needs_human" }, { "type": "score", "correct": true, "kl": 0.20720020867808456, "brier": 0.1141314293, "pred_score": 2.833641, "gold_score": 2.566667, "confidence": 0.86546, "qid": "urgency" } ] }, { "id": "customer_service_000073", "workflow": "customer_service", "decisions": [ { "type": "choice", "correct": false, "kl": 0.4024889081324324, "brier": 0.22567092524699997, "confidence": 0.470707, "qid": "action" }, { "type": "choice", "correct": true, "kl": 0.023541669300416505, "brier": 0.0015429790909999982, "confidence": 0.936693, "qid": "category" }, { "type": "score", "correct": true, "kl": 0.08222079062317561, "brier": 0.016875774091000006, "pred_score": 2.0217, "gold_score": 1.75, "confidence": 0.427498, "qid": "churn_risk" }, { "type": "noul", "correct": false, "kl": 0.06819753877786941, "brier": 0.06472514163199998, "confidence": 0.663229, "qid": "needs_human" }, { "type": "score", "correct": false, "kl": 0.10702550910048464, "brier": 0.08355512864600004, "pred_score": 2.5368990000000005, "gold_score": 2.25, "confidence": 0.609095, "qid": "urgency" } ] }, { "id": "customer_service_000074", "workflow": "customer_service", "decisions": [ { "type": "choice", "correct": true, "kl": 0.18570987962908364, "brier": 0.10157920329699999, "confidence": 0.461767, "qid": "action" }, { "type": "choice", "correct": true, "kl": 0.00949054989889274, "brier": 0.000504251719, "confidence": 0.941684, "qid": "category" }, { "type": "score", "correct": true, "kl": 0.037983227303014386, "brier": 0.017107320136999993, "pred_score": 1.523526, "gold_score": 1.6, "confidence": 0.404291, "qid": "churn_risk" }, { "type": "noul", "correct": true, "kl": 0.07978330005406625, "brier": 0.06923441449800001, "confidence": 0.61061, "qid": "needs_human" }, { "type": "score", "correct": true, "kl": 0.017529766949432424, "brier": 0.00866295712600001, "pred_score": 2.222352, "gold_score": 2.283333, "confidence": 0.595461, "qid": "urgency" } ] }, { "id": "customer_service_000075", "workflow": "customer_service", "decisions": [ { "type": "choice", "correct": true, "kl": 0.09471838312087906, "brier": 0.055098164573000034, "confidence": 0.618624, "qid": "action" }, { "type": "choice", "correct": false, "kl": 0.5106124805169502, "brier": 0.415005726198, "confidence": 0.5121950000000001, "qid": "category" }, { "type": "score", "correct": true, "kl": 0.126372451591178, "brier": 0.073947520589, "pred_score": 2.591871, "gold_score": 2.863333, "confidence": 0.669316, "qid": "churn_risk" }, { "type": "noul", "correct": true, "kl": 0.0938991597778933, "brier": 0.05563247104799999, "confidence": 0.753218, "qid": "needs_human" }, { "type": "score", "correct": true, "kl": 0.0010311914267662776, "brier": 0.00013468784799999944, "pred_score": 2.791305, "gold_score": 2.803333, "confidence": 0.841542, "qid": "urgency" } ] }, { "id": "customer_service_000076", "workflow": "customer_service", "decisions": [ { "type": "choice", "correct": true, "kl": 0.13848050875542436, "brier": 0.07097812395900002, "confidence": 0.604157, "qid": "action" }, { "type": "choice", "correct": true, "kl": 0.018877989958095386, "brier": 0.001539548477000001, "confidence": 0.952154, "qid": "category" }, { "type": "score", "correct": true, "kl": 0.021411845130932305, "brier": 0.016057146338, "pred_score": 2.3013820000000003, "gold_score": 2.24, "confidence": 0.48931, "qid": "churn_risk" }, { "type": "noul", "correct": true, "kl": 0.01467351193236624, "brier": 0.012462099938000001, "confidence": 0.718937, "qid": "needs_human" }, { "type": "score", "correct": false, "kl": 0.012396108258367067, "brier": 0.008448193018999992, "pred_score": 2.276246, "gold_score": 2.376667, "confidence": 0.498393, "qid": "urgency" } ] }, { "id": "customer_service_000077", "workflow": "customer_service", "decisions": [ { "type": "choice", "correct": true, "kl": 0.01937880474095701, "brier": 0.0018506342559999995, "confidence": 0.737167, "qid": "action" }, { "type": "choice", "correct": true, "kl": 0.03817112920689801, "brier": 0.016760707863999982, "confidence": 0.622743, "qid": "category" }, { "type": "score", "correct": true, "kl": 0.05035808585914647, "brier": 0.04073092268600001, "pred_score": 2.526866, "gold_score": 2.683333, "confidence": 0.592545, "qid": "churn_risk" }, { "type": "noul", "correct": true, "kl": 0.0004614713757369361, "brier": 0.00025119369800000223, "confidence": 0.841207, "qid": "needs_human" }, { "type": "score", "correct": true, "kl": 0.027344387364351255, "brier": 0.007809059941, "pred_score": 2.724367, "gold_score": 2.606667, "confidence": 0.764551, "qid": "urgency" } ] }, { "id": "customer_service_000078", "workflow": "customer_service", "decisions": [ { "type": "choice", "correct": true, "kl": 0.02825361757130474, "brier": 0.004691089924000004, "confidence": 0.774936, "qid": "action" }, { "type": "choice", "correct": true, "kl": 0.04315009454189936, "brier": 0.0025507792059999987, "confidence": 0.867588, "qid": "category" }, { "type": "score", "correct": true, "kl": 0.04012619400517087, "brier": 0.029914414398999997, "pred_score": 2.0884840000000002, "gold_score": 2.163333, "confidence": 0.651142, "qid": "churn_risk" }, { "type": "noul", "correct": true, "kl": 0.0011686778057337054, "brier": 0.0006659060179999999, "confidence": 0.821753, "qid": "needs_human" }, { "type": "score", "correct": true, "kl": 0.0016297846382844362, "brier": 0.00030499225800000056, "pred_score": 2.646883, "gold_score": 2.623333, "confidence": 0.714623, "qid": "urgency" } ] }, { "id": "customer_service_000079", "workflow": "customer_service", "decisions": [ { "type": "choice", "correct": true, "kl": 0.05333542218542377, "brier": 0.011752710073999995, "confidence": 0.662455, "qid": "action" }, { "type": "choice", "correct": true, "kl": 0.06129860051388733, "brier": 0.013683600468000011, "confidence": 0.845012, "qid": "category" }, { "type": "score", "correct": true, "kl": 0.08528370152480608, "brier": 0.041078463993, "pred_score": 2.060959, "gold_score": 2.15, "confidence": 0.513319, "qid": "churn_risk" }, { "type": "noul", "correct": true, "kl": 0.039527475489398764, "brier": 0.026467482887999987, "confidence": 0.744962, "qid": "needs_human" }, { "type": "score", "correct": false, "kl": 0.02542829283459412, "brier": 0.007941111366157855, "pred_score": 2.2794757205242795, "gold_score": 2.2, "confidence": 0.4340955659044341, "qid": "urgency" } ] }, { "id": "customer_service_000080", "workflow": "customer_service", "decisions": [ { "type": "choice", "correct": true, "kl": 0.21758474984220375, "brier": 0.03650893705696875, "confidence": 0.4374864374864375, "qid": "action" }, { "type": "choice", "correct": true, "kl": 0.22584803264807551, "brier": 0.10791395695399997, "confidence": 0.867915, "qid": "category" }, { "type": "score", "correct": true, "kl": 0.03064499204507929, "brier": 0.022584374185999994, "pred_score": 1.6909239999999999, "gold_score": 1.816667, "confidence": 0.463361, "qid": "churn_risk" }, { "type": "noul", "correct": true, "kl": 0.06989722786027591, "brier": 0.059500080648, "confidence": 0.627518, "qid": "needs_human" }, { "type": "score", "correct": true, "kl": 0.12932493157344097, "brier": 0.05718130308699999, "pred_score": 2.795922, "gold_score": 2.5, "confidence": 0.846862, "qid": "urgency" } ] }, { "id": "customer_service_000081", "workflow": "customer_service", "decisions": [ { "type": "choice", "correct": true, "kl": 0.06743562380330029, "brier": 0.03793963678600001, "confidence": 0.513193, "qid": "action" }, { "type": "choice", "correct": true, "kl": 0.024567138352367443, "brier": 0.012942119266999997, "confidence": 0.742746, "qid": "category" }, { "type": "score", "correct": true, "kl": 0.029036129142334652, "brier": 0.0034527519659999983, "pred_score": 1.070622, "gold_score": 1.17, "confidence": 0.577893, "qid": "churn_risk" }, { "type": "noul", "correct": true, "kl": 0.023481615596358113, "brier": 0.020830854272000016, "confidence": 0.631277, "qid": "needs_human" }, { "type": "score", "correct": true, "kl": 0.056186691764771576, "brier": 0.01725967418400001, "pred_score": 1.2064599999999999, "gold_score": 1.036667, "confidence": 0.631821, "qid": "urgency" } ] }, { "id": "customer_service_000082", "workflow": "customer_service", "decisions": [ { "type": "choice", "correct": true, "kl": 0.2142721797748276, "brier": 0.04245193229400002, "confidence": 0.68021, "qid": "action" }, { "type": "choice", "correct": true, "kl": 0.08007862696052165, "brier": 0.041436938956000015, "confidence": 0.624623, "qid": "category" }, { "type": "score", "correct": true, "kl": 0.0380237129903985, "brier": 0.023205830898000004, "pred_score": 1.958503, "gold_score": 2.05, "confidence": 0.62646, "qid": "churn_risk" }, { "type": "noul", "correct": true, "kl": 0.0013251574494640324, "brier": 0.001039862407999999, "confidence": 0.739469, "qid": "needs_human" }, { "type": "score", "correct": true, "kl": 0.05345097237920313, "brier": 0.025632967157999998, "pred_score": 1.8225870000000002, "gold_score": 1.933333, "confidence": 0.553745, "qid": "urgency" } ] }, { "id": "customer_service_000083", "workflow": "customer_service", "decisions": [ { "type": "choice", "correct": true, "kl": 0.014307590233028654, "brier": 0.009754401364000003, "confidence": 0.387125, "qid": "action" }, { "type": "choice", "correct": true, "kl": 0.025227463736007157, "brier": 0.006533566480000003, "confidence": 0.822863, "qid": "category" }, { "type": "score", "correct": false, "kl": 0.47550147677307575, "brier": 0.24044599473000006, "pred_score": 1.878565, "gold_score": 1.183333, "confidence": 0.519811, "qid": "churn_risk" }, { "type": "noul", "correct": false, "kl": 0.016265278264195188, "brier": 0.01618956168199999, "confidence": 0.556638, "qid": "needs_human" }, { "type": "score", "correct": true, "kl": 0.0833237829728218, "brier": 0.04312555838699999, "pred_score": 1.844604, "gold_score": 2.133333, "confidence": 0.447666, "qid": "urgency" } ] }, { "id": "customer_service_000084", "workflow": "customer_service", "decisions": [ { "type": "choice", "correct": true, "kl": 0.0605471959240156, "brier": 0.03289932612, "confidence": 0.612835, "qid": "action" }, { "type": "choice", "correct": true, "kl": 0.02238213862844342, "brier": 0.0025448866260000032, "confidence": 0.902169, "qid": "category" }, { "type": "score", "correct": true, "kl": 0.08030872166965651, "brier": 0.018407072749999996, "pred_score": 2.2596890000000003, "gold_score": 2.006667, "confidence": 0.512908, "qid": "churn_risk" }, { "type": "noul", "correct": true, "kl": 0.12644557657729819, "brier": 0.09134307336200002, "confidence": 0.823709, "qid": "needs_human" }, { "type": "score", "correct": true, "kl": 0.04317118518879842, "brier": 0.005715919167353682, "pred_score": 2.3983883983883985, "gold_score": 2.246667, "confidence": 0.4966174966174966, "qid": "urgency" } ] }, { "id": "customer_service_000085", "workflow": "customer_service", "decisions": [ { "type": "choice", "correct": true, "kl": 0.057084938750608986, "brier": 0.017352970003999997, "confidence": 0.675219, "qid": "action" }, { "type": "choice", "correct": true, "kl": 0.06094412076348105, "brier": 0.002364052819, "confidence": 0.934231, "qid": "category" }, { "type": "score", "correct": true, "kl": 0.01241035875779832, "brier": 0.00679753600500001, "pred_score": 2.661293, "gold_score": 2.75, "confidence": 0.72383, "qid": "churn_risk" }, { "type": "noul", "correct": true, "kl": 0.010571205957270825, "brier": 0.005505991922000001, "confidence": 0.827531, "qid": "needs_human" }, { "type": "score", "correct": true, "kl": 0.00193187990556163, "brier": 0.0013462174110000014, "pred_score": 2.5175249999999996, "gold_score": 2.546667, "confidence": 0.599339, "qid": "urgency" } ] }, { "id": "customer_service_000086", "workflow": "customer_service", "decisions": [ { "type": "choice", "correct": true, "kl": 0.04748673978359571, "brier": 0.008506463005999993, "confidence": 0.556718, "qid": "action" }, { "type": "choice", "correct": true, "kl": 0.05365376966280223, "brier": 0.018917378144, "confidence": 0.747451, "qid": "category" }, { "type": "score", "correct": true, "kl": 0.026448350970506032, "brier": 0.005798484954000002, "pred_score": 2.119737, "gold_score": 2.033333, "confidence": 0.60063, "qid": "churn_risk" }, { "type": "noul", "correct": true, "kl": 0.026228120203225194, "brier": 0.015620988258000004, "confidence": 0.845044, "qid": "needs_human" }, { "type": "score", "correct": true, "kl": 0.012203393909046597, "brier": 0.0003404397020000006, "pred_score": 2.044844, "gold_score": 2.083333, "confidence": 0.618166, "qid": "urgency" } ] }, { "id": "customer_service_000087", "workflow": "customer_service", "decisions": [ { "type": "choice", "correct": true, "kl": 0.17397481045587343, "brier": 0.02266889988299999, "confidence": 0.791877, "qid": "action" }, { "type": "choice", "correct": false, "kl": 0.11661291553506359, "brier": 0.08857538579599998, "confidence": 0.631828, "qid": "category" }, { "type": "score", "correct": true, "kl": 0.047886717033596544, "brier": 0.01423024289, "pred_score": 2.079771, "gold_score": 1.933333, "confidence": 0.615516, "qid": "churn_risk" }, { "type": "noul", "correct": true, "kl": 0.0008523753752848768, "brier": 0.0004175472019999978, "confidence": 0.852218, "qid": "needs_human" }, { "type": "score", "correct": true, "kl": 0.04975949446818204, "brier": 0.035280236850000014, "pred_score": 1.788497, "gold_score": 1.666667, "confidence": 0.649831, "qid": "urgency" } ] }, { "id": "customer_service_000088", "workflow": "customer_service", "decisions": [ { "type": "choice", "correct": false, "kl": 0.23341413072310718, "brier": 0.141782152322, "confidence": 0.581611, "qid": "action" }, { "type": "choice", "correct": true, "kl": 0.06662490862655987, "brier": 0.010433710545999987, "confidence": 0.8616, "qid": "category" }, { "type": "score", "correct": true, "kl": 0.09674230145019273, "brier": 0.03659413464299999, "pred_score": 1.995465, "gold_score": 1.716667, "confidence": 0.562226, "qid": "churn_risk" }, { "type": "noul", "correct": false, "kl": 0.13792998836614961, "brier": 0.13290724359199996, "confidence": 0.657786, "qid": "needs_human" }, { "type": "score", "correct": true, "kl": 0.06266258574536722, "brier": 0.023202410634999993, "pred_score": 2.035123, "gold_score": 1.9, "confidence": 0.582786, "qid": "urgency" } ] }, { "id": "customer_service_000089", "workflow": "customer_service", "decisions": [ { "type": "choice", "correct": true, "kl": 0.00738076820918783, "brier": 0.0006908829409999983, "confidence": 0.919616, "qid": "action" }, { "type": "choice", "correct": true, "kl": 0.009401513472443382, "brier": 0.000194207308, "confidence": 0.945456, "qid": "category" }, { "type": "score", "correct": true, "kl": 0.06983828598901812, "brier": 0.009394384153999992, "pred_score": 2.693873, "gold_score": 2.91, "confidence": 0.867059, "qid": "churn_risk" }, { "type": "noul", "correct": true, "kl": 0.02644611627852607, "brier": 0.008761173191999995, "confidence": 0.883814, "qid": "needs_human" }, { "type": "score", "correct": true, "kl": 0.12883888702898413, "brier": 0.05626527808400001, "pred_score": 1.227358, "gold_score": 0.95, "confidence": 0.571819, "qid": "urgency" } ] }, { "id": "customer_service_000090", "workflow": "customer_service", "decisions": [ { "type": "choice", "correct": true, "kl": 0.0700304775283387, "brier": 0.012120828401000002, "confidence": 0.693271, "qid": "action" }, { "type": "choice", "correct": false, "kl": 0.44940234436379145, "brier": 0.2820155222151234, "confidence": 0.4178814178814179, "qid": "category" }, { "type": "score", "correct": true, "kl": 0.002633147569499077, "brier": 7.211747000000013e-05, "pred_score": 2.849087, "gold_score": 2.843333, "confidence": 0.8679, "qid": "churn_risk" }, { "type": "noul", "correct": true, "kl": 0.00031536080955882687, "brier": 0.00017275687200000092, "confidence": 0.839294, "qid": "needs_human" }, { "type": "score", "correct": true, "kl": 0.012621358473246126, "brier": 0.0013952443260000027, "pred_score": 2.674825, "gold_score": 2.633333, "confidence": 0.725858, "qid": "urgency" } ] }, { "id": "customer_service_000091", "workflow": "customer_service", "decisions": [ { "type": "choice", "correct": true, "kl": 0.06198054278689037, "brier": 0.014437319721999985, "confidence": 0.638372, "qid": "action" }, { "type": "choice", "correct": true, "kl": 0.03009782047827717, "brier": 0.003917835765000002, "confidence": 0.916947, "qid": "category" }, { "type": "score", "correct": true, "kl": 0.06225468787652777, "brier": 0.02709264799799999, "pred_score": 1.995088, "gold_score": 2.25, "confidence": 0.522965, "qid": "churn_risk" }, { "type": "noul", "correct": true, "kl": 0.04853905909609611, "brier": 0.03124750004999999, "confidence": 0.751672, "qid": "needs_human" }, { "type": "score", "correct": false, "kl": 0.05963434702499415, "brier": 0.045979474510179495, "pred_score": 2.2410982410982414, "gold_score": 2.05, "confidence": 0.4304644304644305, "qid": "urgency" } ] }, { "id": "customer_service_000092", "workflow": "customer_service", "decisions": [ { "type": "choice", "correct": true, "kl": 0.021155363546328736, "brier": 0.008209559723000004, "confidence": 0.525952, "qid": "action" }, { "type": "choice", "correct": true, "kl": 0.04113051778738195, "brier": 0.005269973398999997, "confidence": 0.946282, "qid": "category" }, { "type": "score", "correct": true, "kl": 0.08174145163585164, "brier": 0.023880828277999994, "pred_score": 1.7570700000000001, "gold_score": 1.933333, "confidence": 0.636008, "qid": "churn_risk" }, { "type": "noul", "correct": true, "kl": 3.113598437421263e-06, "brier": 2.3805620000001587e-06, "confidence": 0.742242, "qid": "needs_human" }, { "type": "score", "correct": true, "kl": 0.13215741654038327, "brier": 0.057409125846, "pred_score": 1.880983, "gold_score": 2.21, "confidence": 0.584141, "qid": "urgency" } ] }, { "id": "customer_service_000093", "workflow": "customer_service", "decisions": [ { "type": "choice", "correct": true, "kl": 0.003295326424004875, "brier": 0.00012795468399999982, "confidence": 0.77806, "qid": "action" }, { "type": "choice", "correct": true, "kl": 0.1038592167090496, "brier": 0.008598959281999996, "confidence": 0.859367, "qid": "category" }, { "type": "score", "correct": true, "kl": 0.05869580230661545, "brier": 0.03577840007399998, "pred_score": 2.6620600000000003, "gold_score": 2.833333, "confidence": 0.705523, "qid": "churn_risk" }, { "type": "noul", "correct": true, "kl": 0.0003549559223787607, "brier": 0.00015781537800000153, "confidence": 0.87555, "qid": "needs_human" }, { "type": "score", "correct": true, "kl": 0.018385171154147256, "brier": 0.005577465049999991, "pred_score": 2.835303, "gold_score": 2.743333, "confidence": 0.86171, "qid": "urgency" } ] }, { "id": "customer_service_000094", "workflow": "customer_service", "decisions": [ { "type": "choice", "correct": false, "kl": 0.7335919673533543, "brier": 0.49169938608899993, "confidence": 0.555225, "qid": "action" }, { "type": "choice", "correct": true, "kl": 0.014989657546287694, "brier": 0.0009492991840000016, "confidence": 0.868467, "qid": "category" }, { "type": "score", "correct": true, "kl": 0.004000781941197091, "brier": 0.0010509806839999994, "pred_score": 1.8984400000000001, "gold_score": 1.853333, "confidence": 0.6339130000000001, "qid": "churn_risk" }, { "type": "noul", "correct": true, "kl": 0.05265253845414605, "brier": 0.04674336576799999, "confidence": 0.712878, "qid": "needs_human" }, { "type": "score", "correct": true, "kl": 0.011245114979204503, "brier": 0.0010055994439999965, "pred_score": 2.7709070000000002, "gold_score": 2.826667, "confidence": 0.824617, "qid": "urgency" } ] }, { "id": "customer_service_000095", "workflow": "customer_service", "decisions": [ { "type": "choice", "correct": false, "kl": 0.24946544943892746, "brier": 0.13508463175281787, "confidence": 0.3454696545303454, "qid": "action" }, { "type": "choice", "correct": true, "kl": 0.08413059737006295, "brier": 0.011372351675000007, "confidence": 0.934261, "qid": "category" }, { "type": "score", "correct": true, "kl": 0.009708614132830057, "brier": 0.0023534990420000027, "pred_score": 1.39334, "gold_score": 1.366667, "confidence": 0.516536, "qid": "churn_risk" }, { "type": "noul", "correct": true, "kl": 0.037693084236662636, "brier": 0.03711123192200002, "confidence": 0.507114, "qid": "needs_human" }, { "type": "score", "correct": true, "kl": 0.02716108110812311, "brier": 0.018950527842, "pred_score": 1.507346, "gold_score": 1.366667, "confidence": 0.506401, "qid": "urgency" } ] }, { "id": "customer_service_000096", "workflow": "customer_service", "decisions": [ { "type": "choice", "correct": false, "kl": 0.13364671010180887, "brier": 0.048812726917, "confidence": 0.41552300000000003, "qid": "action" }, { "type": "choice", "correct": true, "kl": 0.04428901180769477, "brier": 0.011217859310000002, "confidence": 0.835735, "qid": "category" }, { "type": "score", "correct": true, "kl": 0.054200938738492695, "brier": 0.01282225644199998, "pred_score": 2.7307930000000002, "gold_score": 2.866667, "confidence": 0.773808, "qid": "churn_risk" }, { "type": "noul", "correct": true, "kl": 2.5906123509109643e-05, "brier": 2.2337928000000154e-05, "confidence": 0.686675, "qid": "needs_human" }, { "type": "score", "correct": true, "kl": 0.06395311340616519, "brier": 0.041501834390000034, "pred_score": 2.608629, "gold_score": 2.516667, "confidence": 0.659699, "qid": "urgency" } ] }, { "id": "customer_service_000097", "workflow": "customer_service", "decisions": [ { "type": "choice", "correct": true, "kl": 0.04139706385183211, "brier": 0.012809480809999996, "confidence": 0.772057, "qid": "action" }, { "type": "choice", "correct": true, "kl": 0.03063362230625588, "brier": 0.0030080609119999984, "confidence": 0.898964, "qid": "category" }, { "type": "score", "correct": true, "kl": 0.050698633527176545, "brier": 0.035362312982, "pred_score": 2.539039, "gold_score": 2.716667, "confidence": 0.621401, "qid": "churn_risk" }, { "type": "noul", "correct": true, "kl": 9.30346127517767e-07, "brier": 4.032019999999341e-07, "confidence": 0.876218, "qid": "needs_human" }, { "type": "score", "correct": true, "kl": 0.009231002077113956, "brier": 0.006914762077999996, "pred_score": 2.405277, "gold_score": 2.326667, "confidence": 0.480853, "qid": "urgency" } ] }, { "id": "customer_service_000098", "workflow": "customer_service", "decisions": [ { "type": "choice", "correct": false, "kl": 0.3969185464361922, "brier": 0.17883147093899998, "confidence": 0.524944, "qid": "action" }, { "type": "choice", "correct": true, "kl": 0.09169589812618392, "brier": 0.010599399406000003, "confidence": 0.934225, "qid": "category" }, { "type": "score", "correct": true, "kl": 0.13742291531626147, "brier": 0.07449214385999998, "pred_score": 2.259009, "gold_score": 1.983333, "confidence": 0.476778, "qid": "churn_risk" }, { "type": "noul", "correct": true, "kl": 0.05785919534345904, "brier": 0.052774156962000024, "confidence": 0.695774, "qid": "needs_human" }, { "type": "score", "correct": true, "kl": 0.011357630227708502, "brier": 0.0006930935339999995, "pred_score": 2.777233, "gold_score": 2.816667, "confidence": 0.821701, "qid": "urgency" } ] }, { "id": "customer_service_000099", "workflow": "customer_service", "decisions": [ { "type": "choice", "correct": true, "kl": 0.3135563990872945, "brier": 0.08508527193399998, "confidence": 0.692585, "qid": "action" }, { "type": "choice", "correct": true, "kl": 0.10104581947241445, "brier": 0.036337200922, "confidence": 0.7839700000000001, "qid": "category" }, { "type": "score", "correct": false, "kl": 0.5197002931629767, "brier": 0.250987460622, "pred_score": 1.305865, "gold_score": 0.616667, "confidence": 0.486135, "qid": "churn_risk" }, { "type": "noul", "correct": true, "kl": 0.006740445421186819, "brier": 0.006226833608000003, "confidence": 0.655798, "qid": "needs_human" }, { "type": "score", "correct": true, "kl": 0.0009819750321380348, "brier": 0.00040266194999999866, "pred_score": 2.763977, "gold_score": 2.776667, "confidence": 0.819424, "qid": "urgency" } ] }, { "id": "invoice_processing_000000", "workflow": "invoice_processing", "decisions": [ { "type": "score", "correct": false, "kl": 0.26732134530562424, "brier": 0.131573715614, "pred_score": 1.245022, "gold_score": 1.303333, "confidence": 0.497498, "qid": "discrepancy_severity" }, { "type": "choice", "correct": true, "kl": 0.034168185570325985, "brier": 0.006918344557156432, "confidence": 0.46463246463246466, "qid": "disposition" }, { "type": "noul", "correct": true, "kl": 4.582423320859141e-05, "brier": 6.272881999999865e-06, "confidence": 0.965104, "qid": "duplicate" }, { "type": "noul", "correct": false, "kl": 0.030591516620649184, "brier": 0.030018870338000006, "confidence": 0.602513, "qid": "matches_order" }, { "type": "score", "correct": true, "kl": 0.012169251153398654, "brier": 0.007507173853000003, "pred_score": 2.0452879999999998, "gold_score": 1.983333, "confidence": 0.508687, "qid": "urgency" } ] }, { "id": "invoice_processing_000001", "workflow": "invoice_processing", "decisions": [ { "type": "score", "correct": true, "kl": 0.006048604613815622, "brier": 0.0017320075370000011, "pred_score": 2.7584739999999996, "gold_score": 2.79, "confidence": 0.822286, "qid": "discrepancy_severity" }, { "type": "choice", "correct": true, "kl": 0.1402879272142539, "brier": 0.09555088877600001, "confidence": 0.690841, "qid": "disposition" }, { "type": "noul", "correct": true, "kl": 5.714475650079893e-05, "brier": 6.393888000000034e-06, "confidence": 0.971788, "qid": "duplicate" }, { "type": "noul", "correct": true, "kl": 0.00011899421946438013, "brier": 2.1661362000000303e-05, "confidence": 0.953291, "qid": "matches_order" }, { "type": "score", "correct": false, "kl": 0.23916649604708057, "brier": 0.19163501589299997, "pred_score": 1.349017, "gold_score": 1.75, "confidence": 0.608065, "qid": "urgency" } ] }, { "id": "invoice_processing_000002", "workflow": "invoice_processing", "decisions": [ { "type": "score", "correct": true, "kl": 0.06521778929268726, "brier": 0.009382192765999994, "pred_score": 0.195055, "gold_score": 0.026667, "confidence": 0.895976, "qid": "discrepancy_severity" }, { "type": "choice", "correct": true, "kl": 0.06249214061809548, "brier": 0.015227596381999994, "confidence": 0.859048, "qid": "disposition" }, { "type": "noul", "correct": true, "kl": 0.00931677187614445, "brier": 0.0011342131920000001, "confidence": 0.959519, "qid": "duplicate" }, { "type": "noul", "correct": true, "kl": 0.04049348974956429, "brier": 0.007562532128, "confidence": 0.925175, "qid": "matches_order" }, { "type": "score", "correct": false, "kl": 0.29806945329650114, "brier": 0.202228071079, "pred_score": 1.701232, "gold_score": 2.0, "confidence": 0.401934, "qid": "urgency" } ] }, { "id": "invoice_processing_000003", "workflow": "invoice_processing", "decisions": [ { "type": "score", "correct": true, "kl": 0.09039006346117426, "brier": 0.030437938842000013, "pred_score": 2.104118, "gold_score": 2.0, "confidence": 0.72128, "qid": "discrepancy_severity" }, { "type": "choice", "correct": false, "kl": 0.23686143320252495, "brier": 0.18874238779000002, "confidence": 0.668111, "qid": "disposition" }, { "type": "noul", "correct": true, "kl": 0.001763102282130162, "brier": 0.00022353436800000057, "confidence": 0.970572, "qid": "duplicate" }, { "type": "noul", "correct": true, "kl": 0.054315676184898, "brier": 0.007751121032000005, "confidence": 0.978921, "qid": "matches_order" }, { "type": "score", "correct": true, "kl": 0.047052367433461716, "brier": 0.01482006191516159, "pred_score": 2.1426618573381426, "gold_score": 2.233333, "confidence": 0.49994350005649996, "qid": "urgency" } ] }, { "id": "invoice_processing_000004", "workflow": "invoice_processing", "decisions": [ { "type": "score", "correct": true, "kl": 0.05731655962321669, "brier": 0.008685314073000003, "pred_score": 2.784437, "gold_score": 2.8, "confidence": 0.862083, "qid": "discrepancy_severity" }, { "type": "choice", "correct": true, "kl": 0.22363210142386633, "brier": 0.11810406195800001, "confidence": 0.664896, "qid": "disposition" }, { "type": "noul", "correct": true, "kl": 1.8188630489141021e-06, "brier": 1.1809799999999679e-07, "confidence": 0.983576, "qid": "duplicate" }, { "type": "noul", "correct": true, "kl": 0.0008356325685103718, "brier": 0.00019792540799999897, "confidence": 0.933385, "qid": "matches_order" }, { "type": "score", "correct": false, "kl": 0.32265206527930157, "brier": 0.15423728560799999, "pred_score": 1.432169, "gold_score": 1.983333, "confidence": 0.511597, "qid": "urgency" } ] }, { "id": "invoice_processing_000005", "workflow": "invoice_processing", "decisions": [ { "type": "score", "correct": false, "kl": 0.44327490535423875, "brier": 0.24695942039843208, "pred_score": 1.5976835976835977, "gold_score": 1.833333, "confidence": 0.3951863951863952, "qid": "discrepancy_severity" }, { "type": "choice", "correct": true, "kl": 0.09639709919175662, "brier": 0.05610679715559785, "confidence": 0.413026413026413, "qid": "disposition" }, { "type": "noul", "correct": true, "kl": 0.008976477165604322, "brier": 0.0011501767220000016, "confidence": 0.973981, "qid": "duplicate" }, { "type": "noul", "correct": true, "kl": 0.08147940599844045, "brier": 0.07719149152800002, "confidence": 0.536875, "qid": "matches_order" }, { "type": "score", "correct": true, "kl": 0.06614019771510654, "brier": 0.02314734515599999, "pred_score": 1.9311830000000003, "gold_score": 1.866667, "confidence": 0.5555720000000001, "qid": "urgency" } ] }, { "id": "invoice_processing_000006", "workflow": "invoice_processing", "decisions": [ { "type": "score", "correct": true, "kl": 0.3172781142566756, "brier": 0.13403977421399998, "pred_score": 1.868123, "gold_score": 1.45, "confidence": 0.720831, "qid": "discrepancy_severity" }, { "type": "choice", "correct": true, "kl": 0.229345523319303, "brier": 0.11057424420080397, "confidence": 0.45685345685345685, "qid": "disposition" }, { "type": "noul", "correct": true, "kl": 0.002350273903359108, "brier": 0.00018447363199999899, "confidence": 0.982937, "qid": "duplicate" }, { "type": "noul", "correct": true, "kl": 0.1543964751626183, "brier": 0.10508269992200002, "confidence": 0.845886, "qid": "matches_order" }, { "type": "score", "correct": true, "kl": 0.03251644128480818, "brier": 0.020106770144999994, "pred_score": 1.37178, "gold_score": 1.533333, "confidence": 0.593239, "qid": "urgency" } ] }, { "id": "invoice_processing_000007", "workflow": "invoice_processing", "decisions": [ { "type": "score", "correct": true, "kl": 0.27301480578088766, "brier": 0.14915067775799998, "pred_score": 2.407362, "gold_score": 2.903333, "confidence": 0.601167, "qid": "discrepancy_severity" }, { "type": "choice", "correct": true, "kl": 0.05603302090970503, "brier": 0.014745792653999994, "confidence": 0.662787, "qid": "disposition" }, { "type": "noul", "correct": true, "kl": 0.006225536580835746, "brier": 0.0004769107279999986, "confidence": 0.974558, "qid": "duplicate" }, { "type": "noul", "correct": true, "kl": 0.05992540151034226, "brier": 0.014877435008, "confidence": 0.897085, "qid": "matches_order" }, { "type": "score", "correct": true, "kl": 0.02052936821376347, "brier": 0.016028652834338394, "pred_score": 2.152020152020152, "gold_score": 2.106667, "confidence": 0.4901594901594902, "qid": "urgency" } ] }, { "id": "invoice_processing_000008", "workflow": "invoice_processing", "decisions": [ { "type": "score", "correct": true, "kl": 0.08535381032920102, "brier": 0.060137723541999984, "pred_score": 2.579049, "gold_score": 2.793333, "confidence": 0.649864, "qid": "discrepancy_severity" }, { "type": "choice", "correct": false, "kl": 0.11180828651985124, "brier": 0.09060746000599999, "confidence": 0.570201, "qid": "disposition" }, { "type": "noul", "correct": true, "kl": 3.486534356210978e-05, "brier": 5.062562000000035e-06, "confidence": 0.961742, "qid": "duplicate" }, { "type": "noul", "correct": true, "kl": 0.007976262026266143, "brier": 0.0018613540979999974, "confidence": 0.947174, "qid": "matches_order" }, { "type": "score", "correct": true, "kl": 0.014428924057899955, "brier": 0.002232985373011791, "pred_score": 1.5676615676615677, "gold_score": 1.666667, "confidence": 0.4779024779024779, "qid": "urgency" } ] }, { "id": "invoice_processing_000009", "workflow": "invoice_processing", "decisions": [ { "type": "score", "correct": false, "kl": 0.05335621112150844, "brier": 0.040600460293753166, "pred_score": 1.0446029553970448, "gold_score": 1.333333, "confidence": 0.4263955736044264, "qid": "discrepancy_severity" }, { "type": "choice", "correct": false, "kl": 0.11923022555000226, "brier": 0.05596649928935608, "confidence": 0.4161925838074162, "qid": "disposition" }, { "type": "noul", "correct": true, "kl": 0.0004057406053652188, "brier": 4.160544199999932e-05, "confidence": 0.972106, "qid": "duplicate" }, { "type": "noul", "correct": true, "kl": 0.006885045063404115, "brier": 0.006081280327999993, "confidence": 0.688475, "qid": "matches_order" }, { "type": "score", "correct": true, "kl": 0.17913095040059068, "brier": 0.11890920075299997, "pred_score": 1.4912779999999999, "gold_score": 1.133333, "confidence": 0.482922, "qid": "urgency" } ] }, { "id": "invoice_processing_000010", "workflow": "invoice_processing", "decisions": [ { "type": "score", "correct": true, "kl": 0.05604206989449994, "brier": 0.009751044277999993, "pred_score": 0.207564, "gold_score": 0.040333, "confidence": 0.887452, "qid": "discrepancy_severity" }, { "type": "choice", "correct": true, "kl": 0.019517280953006786, "brier": 0.005454637559999999, "confidence": 0.842612, "qid": "disposition" }, { "type": "noul", "correct": true, "kl": 0.01326891266225963, "brier": 0.0015599463679999977, "confidence": 0.958739, "qid": "duplicate" }, { "type": "noul", "correct": true, "kl": 0.025322698585755563, "brier": 0.005196085681999999, "confidence": 0.925696, "qid": "matches_order" }, { "type": "score", "correct": true, "kl": 0.09821788355595447, "brier": 0.07224597367400001, "pred_score": 1.762847, "gold_score": 1.873333, "confidence": 0.446914, "qid": "urgency" } ] }, { "id": "invoice_processing_000011", "workflow": "invoice_processing", "decisions": [ { "type": "score", "correct": true, "kl": 0.11629353856999726, "brier": 0.021948982662999997, "pred_score": 0.245569, "gold_score": 0.008667, "confidence": 0.868631, "qid": "discrepancy_severity" }, { "type": "choice", "correct": true, "kl": 0.12302749811057404, "brier": 0.031106434698000018, "confidence": 0.835652, "qid": "disposition" }, { "type": "noul", "correct": true, "kl": 0.02222899957129932, "brier": 0.0017288376020000006, "confidence": 0.967266, "qid": "duplicate" }, { "type": "noul", "correct": true, "kl": 0.09706058854907854, "brier": 0.02117311576199999, "confidence": 0.893776, "qid": "matches_order" }, { "type": "score", "correct": true, "kl": 0.4069516805474722, "brier": 0.25799014812800003, "pred_score": 1.756613, "gold_score": 1.15, "confidence": 0.38462, "qid": "urgency" } ] }, { "id": "invoice_processing_000012", "workflow": "invoice_processing", "decisions": [ { "type": "score", "correct": true, "kl": 0.40508608688160386, "brier": 0.12176867140300003, "pred_score": 0.35927, "gold_score": 1.05, "confidence": 0.815114, "qid": "discrepancy_severity" }, { "type": "choice", "correct": false, "kl": 0.5507756007016794, "brier": 0.307579516406, "confidence": 0.641269, "qid": "disposition" }, { "type": "noul", "correct": false, "kl": 0.3825791496381468, "brier": 0.3618996918479999, "confidence": 0.695382, "qid": "duplicate" }, { "type": "noul", "correct": true, "kl": 0.03978717158990146, "brier": 0.009630776897999991, "confidence": 0.907274, "qid": "matches_order" }, { "type": "score", "correct": true, "kl": 0.023436256338386557, "brier": 0.01595910879399999, "pred_score": 2.326157, "gold_score": 2.183333, "confidence": 0.465684, "qid": "urgency" } ] }, { "id": "invoice_processing_000013", "workflow": "invoice_processing", "decisions": [ { "type": "score", "correct": true, "kl": 0.17519738063882642, "brier": 0.09115972709799995, "pred_score": 2.523169, "gold_score": 2.866667, "confidence": 0.623519, "qid": "discrepancy_severity" }, { "type": "choice", "correct": true, "kl": 0.06293351380767874, "brier": 0.03477647340199999, "confidence": 0.702616, "qid": "disposition" }, { "type": "noul", "correct": true, "kl": 0.0006925035970076895, "brier": 9.44487679999999e-05, "confidence": 0.966872, "qid": "duplicate" }, { "type": "noul", "correct": true, "kl": 0.010399512563218976, "brier": 0.0013059116180000026, "confidence": 0.95778, "qid": "matches_order" }, { "type": "score", "correct": false, "kl": 0.16325696556996502, "brier": 0.12216389534326695, "pred_score": 1.55998055998056, "gold_score": 1.233333, "confidence": 0.4370494370494371, "qid": "urgency" } ] }, { "id": "invoice_processing_000014", "workflow": "invoice_processing", "decisions": [ { "type": "score", "correct": false, "kl": 0.8596792575957689, "brier": 0.479883971722, "pred_score": 0.44546800000000003, "gold_score": 1.216667, "confidence": 0.776153, "qid": "discrepancy_severity" }, { "type": "choice", "correct": true, "kl": 0.26266719758270235, "brier": 0.118319746618, "confidence": 0.689698, "qid": "disposition" }, { "type": "noul", "correct": true, "kl": 0.008669513308522114, "brier": 0.002307650048, "confidence": 0.916032, "qid": "duplicate" }, { "type": "noul", "correct": true, "kl": 0.12347380236347999, "brier": 0.09171616205000002, "confidence": 0.814145, "qid": "matches_order" }, { "type": "score", "correct": true, "kl": 0.2013682344031083, "brier": 0.10389409860962909, "pred_score": 1.727916272083728, "gold_score": 2.133333, "confidence": 0.403994596005404, "qid": "urgency" } ] }, { "id": "invoice_processing_000015", "workflow": "invoice_processing", "decisions": [ { "type": "score", "correct": true, "kl": 0.13981371605560566, "brier": 0.034150802306, "pred_score": 0.333602, "gold_score": 0.023333, "confidence": 0.828564, "qid": "discrepancy_severity" }, { "type": "choice", "correct": true, "kl": 0.15950362015414796, "brier": 0.06302974516600003, "confidence": 0.732103, "qid": "disposition" }, { "type": "noul", "correct": true, "kl": 0.04791602560574738, "brier": 0.011614404049999996, "confidence": 0.903795, "qid": "duplicate" }, { "type": "noul", "correct": true, "kl": 0.09598322620152003, "brier": 0.023578364168000014, "confidence": 0.884755, "qid": "matches_order" }, { "type": "score", "correct": true, "kl": 0.12423196855951638, "brier": 0.06528035665, "pred_score": 1.648335, "gold_score": 1.833333, "confidence": 0.452954, "qid": "urgency" } ] }, { "id": "invoice_processing_000016", "workflow": "invoice_processing", "decisions": [ { "type": "score", "correct": true, "kl": 0.022910003239824235, "brier": 0.0011347887899999997, "pred_score": 2.8200220000000003, "gold_score": 2.893333, "confidence": 0.874279, "qid": "discrepancy_severity" }, { "type": "choice", "correct": true, "kl": 0.009641942586588369, "brier": 0.0013087569440000015, "confidence": 0.710707, "qid": "disposition" }, { "type": "noul", "correct": true, "kl": 0.0009408638296481923, "brier": 8.204804999999995e-05, "confidence": 0.979738, "qid": "duplicate" }, { "type": "noul", "correct": true, "kl": 0.00566650971727805, "brier": 0.0006975859520000011, "confidence": 0.961324, "qid": "matches_order" }, { "type": "score", "correct": true, "kl": 0.03404442221948306, "brier": 0.017954493846, "pred_score": 1.820887, "gold_score": 2.0, "confidence": 0.426563, "qid": "urgency" } ] }, { "id": "invoice_processing_000017", "workflow": "invoice_processing", "decisions": [ { "type": "score", "correct": false, "kl": 0.6701405855034503, "brier": 0.5353646242529999, "pred_score": 2.7121120000000003, "gold_score": 2.13, "confidence": 0.771031, "qid": "discrepancy_severity" }, { "type": "choice", "correct": true, "kl": 0.059430072290025, "brier": 0.036422261064, "confidence": 0.693177, "qid": "disposition" }, { "type": "noul", "correct": true, "kl": 0.0002792384386445042, "brier": 3.176045000000012e-05, "confidence": 0.969348, "qid": "duplicate" }, { "type": "noul", "correct": true, "kl": 0.03092692803696538, "brier": 0.005958607777999993, "confidence": 0.964583, "qid": "matches_order" }, { "type": "score", "correct": true, "kl": 0.0572742761139587, "brier": 0.03197127017, "pred_score": 1.2918649999999998, "gold_score": 1.103333, "confidence": 0.631497, "qid": "urgency" } ] }, { "id": "invoice_processing_000018", "workflow": "invoice_processing", "decisions": [ { "type": "score", "correct": true, "kl": 0.2692162151833019, "brier": 0.20485735633099997, "pred_score": 2.440916, "gold_score": 2.816667, "confidence": 0.533904, "qid": "discrepancy_severity" }, { "type": "choice", "correct": false, "kl": 0.022683697471481593, "brier": 0.016014961234999993, "confidence": 0.53416, "qid": "disposition" }, { "type": "noul", "correct": true, "kl": 0.002728498114388078, "brier": 0.0003733551379999994, "confidence": 0.95967, "qid": "duplicate" }, { "type": "noul", "correct": true, "kl": 0.002379675672776905, "brier": 0.0006031559119999998, "confidence": 0.925967, "qid": "matches_order" }, { "type": "score", "correct": false, "kl": 0.03707436391250177, "brier": 0.023944297437999994, "pred_score": 2.139255, "gold_score": 2.326667, "confidence": 0.490114, "qid": "urgency" } ] }, { "id": "invoice_processing_000019", "workflow": "invoice_processing", "decisions": [ { "type": "score", "correct": false, "kl": 0.15451786513418048, "brier": 0.14259217120200005, "pred_score": 2.331053, "gold_score": 2.61, "confidence": 0.546062, "qid": "discrepancy_severity" }, { "type": "choice", "correct": true, "kl": 0.043777811138144836, "brier": 0.02325834361400001, "confidence": 0.632612, "qid": "disposition" }, { "type": "noul", "correct": true, "kl": 0.002664054657468589, "brier": 0.00029325576200000083, "confidence": 0.975442, "qid": "duplicate" }, { "type": "noul", "correct": true, "kl": 0.0315241755925801, "brier": 0.0027618798419999997, "confidence": 0.987161, "qid": "matches_order" }, { "type": "score", "correct": true, "kl": 0.011019460934786146, "brier": 0.0070697702190000026, "pred_score": 2.013891, "gold_score": 2.04, "confidence": 0.564404, "qid": "urgency" } ] }, { "id": "invoice_processing_000020", "workflow": "invoice_processing", "decisions": [ { "type": "score", "correct": true, "kl": 0.0930912542633002, "brier": 0.01419249607199999, "pred_score": 0.216217, "gold_score": 0.013333, "confidence": 0.885753, "qid": "discrepancy_severity" }, { "type": "choice", "correct": true, "kl": 0.07438427754242022, "brier": 0.016765603573999987, "confidence": 0.862156, "qid": "disposition" }, { "type": "noul", "correct": true, "kl": 0.01089260574031217, "brier": 0.0007928161999999994, "confidence": 0.973423, "qid": "duplicate" }, { "type": "noul", "correct": true, "kl": 0.06546323271040447, "brier": 0.012534511112000012, "confidence": 0.914167, "qid": "matches_order" }, { "type": "score", "correct": false, "kl": 0.1264475956717848, "brier": 0.037772075467868524, "pred_score": 1.557989557989558, "gold_score": 1.61, "confidence": 0.44946944946944944, "qid": "urgency" } ] }, { "id": "invoice_processing_000021", "workflow": "invoice_processing", "decisions": [ { "type": "score", "correct": true, "kl": 0.1305968975312004, "brier": 0.047756501353000004, "pred_score": 0.7025669999999999, "gold_score": 0.233667, "confidence": 0.703301, "qid": "discrepancy_severity" }, { "type": "choice", "correct": true, "kl": 0.2987856432732569, "brier": 0.17858803515700936, "confidence": 0.42019357980642014, "qid": "disposition" }, { "type": "noul", "correct": false, "kl": 0.20060264287335505, "brier": 0.192016523808, "confidence": 0.536519, "qid": "duplicate" }, { "type": "noul", "correct": true, "kl": 0.08447748094357657, "brier": 0.032763904127999996, "confidence": 0.838341, "qid": "matches_order" }, { "type": "score", "correct": true, "kl": 0.2787469720063987, "brier": 0.137197234378, "pred_score": 1.153651, "gold_score": 0.800333, "confidence": 0.65583, "qid": "urgency" } ] }, { "id": "invoice_processing_000022", "workflow": "invoice_processing", "decisions": [ { "type": "score", "correct": true, "kl": 0.35674898006576433, "brier": 0.16551384504448896, "pred_score": 1.1494611494611495, "gold_score": 0.456667, "confidence": 0.3831043831043831, "qid": "discrepancy_severity" }, { "type": "choice", "correct": true, "kl": 0.3444848312477407, "brier": 0.232889670994, "confidence": 0.373655, "qid": "disposition" }, { "type": "noul", "correct": true, "kl": 0.023066878470528154, "brier": 0.005343918961999993, "confidence": 0.918309, "qid": "duplicate" }, { "type": "noul", "correct": true, "kl": 0.13712365931816434, "brier": 0.099064126728, "confidence": 0.674109, "qid": "matches_order" }, { "type": "score", "correct": true, "kl": 0.06669494548298889, "brier": 0.03281931266988785, "pred_score": 1.6880943119056884, "gold_score": 1.876667, "confidence": 0.43968456031543973, "qid": "urgency" } ] }, { "id": "invoice_processing_000023", "workflow": "invoice_processing", "decisions": [ { "type": "score", "correct": true, "kl": 0.10930308138368959, "brier": 0.027533405017000006, "pred_score": 0.504602, "gold_score": 0.143333, "confidence": 0.777942, "qid": "discrepancy_severity" }, { "type": "choice", "correct": true, "kl": 0.13410139826203535, "brier": 0.031308999476, "confidence": 0.58993, "qid": "disposition" }, { "type": "noul", "correct": false, "kl": 0.1382143926933488, "brier": 0.13172715264199997, "confidence": 0.673306, "qid": "duplicate" }, { "type": "noul", "correct": true, "kl": 0.05624098573426352, "brier": 0.019484169607999986, "confidence": 0.864631, "qid": "matches_order" }, { "type": "score", "correct": true, "kl": 0.07826002313181463, "brier": 0.023652663754000016, "pred_score": 1.197929, "gold_score": 0.976667, "confidence": 0.70285, "qid": "urgency" } ] }, { "id": "invoice_processing_000024", "workflow": "invoice_processing", "decisions": [ { "type": "score", "correct": true, "kl": 0.06952321935201361, "brier": 0.015044024330999991, "pred_score": 0.270797, "gold_score": 0.053333, "confidence": 0.860052, "qid": "discrepancy_severity" }, { "type": "choice", "correct": true, "kl": 0.09671824257835551, "brier": 0.029817984694000023, "confidence": 0.809387, "qid": "disposition" }, { "type": "noul", "correct": true, "kl": 0.027006296505711947, "brier": 0.004703336072000004, "confidence": 0.934839, "qid": "duplicate" }, { "type": "noul", "correct": true, "kl": 0.05246962254370964, "brier": 0.012323872007999993, "confidence": 0.904502, "qid": "matches_order" }, { "type": "score", "correct": false, "kl": 0.016376327153397627, "brier": 0.004673166163999998, "pred_score": 1.5953140000000001, "gold_score": 1.616667, "confidence": 0.438682, "qid": "urgency" } ] }, { "id": "invoice_processing_000025", "workflow": "invoice_processing", "decisions": [ { "type": "score", "correct": false, "kl": 0.20429027348899534, "brier": 0.18519843887800005, "pred_score": 2.362647, "gold_score": 2.68, "confidence": 0.517395, "qid": "discrepancy_severity" }, { "type": "choice", "correct": true, "kl": 0.030203871938514266, "brier": 0.012987332378000012, "confidence": 0.613003, "qid": "disposition" }, { "type": "noul", "correct": true, "kl": 0.005042855630078157, "brier": 0.00036396020000000005, "confidence": 0.97651, "qid": "duplicate" }, { "type": "noul", "correct": true, "kl": 0.002521448793106347, "brier": 0.0001681778000000002, "confidence": 0.985837, "qid": "matches_order" }, { "type": "score", "correct": true, "kl": 0.005793244402509504, "brier": 0.002623871469999998, "pred_score": 1.796948, "gold_score": 1.866667, "confidence": 0.578368, "qid": "urgency" } ] }, { "id": "invoice_processing_000026", "workflow": "invoice_processing", "decisions": [ { "type": "score", "correct": true, "kl": 0.2774570616391613, "brier": 0.097474207941, "pred_score": 0.329166, "gold_score": 0.98, "confidence": 0.84578, "qid": "discrepancy_severity" }, { "type": "choice", "correct": true, "kl": 0.20969486696754602, "brier": 0.125774742376, "confidence": 0.707821, "qid": "disposition" }, { "type": "noul", "correct": false, "kl": 0.20076084939509878, "brier": 0.17623390804999994, "confidence": 0.753512, "qid": "duplicate" }, { "type": "noul", "correct": true, "kl": 0.027366033187681722, "brier": 0.00711338208800001, "confidence": 0.907029, "qid": "matches_order" }, { "type": "score", "correct": true, "kl": 0.06290214280235717, "brier": 0.012914490804000004, "pred_score": 1.524745, "gold_score": 1.45, "confidence": 0.472199, "qid": "urgency" } ] }, { "id": "invoice_processing_000027", "workflow": "invoice_processing", "decisions": [ { "type": "score", "correct": false, "kl": 0.31655098191684333, "brier": 0.1364956386454421, "pred_score": 1.3516126483873516, "gold_score": 0.616667, "confidence": 0.32493967506032495, "qid": "discrepancy_severity" }, { "type": "choice", "correct": false, "kl": 0.2880131431656535, "brier": 0.16544981473939707, "confidence": 0.38625638625638625, "qid": "disposition" }, { "type": "noul", "correct": true, "kl": 0.055414098674951284, "brier": 0.049664173448, "confidence": 0.707582, "qid": "duplicate" }, { "type": "noul", "correct": true, "kl": 0.10317755099856765, "brier": 0.08204895404999998, "confidence": 0.647455, "qid": "matches_order" }, { "type": "score", "correct": true, "kl": 0.08129497299037158, "brier": 0.05848300607, "pred_score": 1.46107, "gold_score": 1.216667, "confidence": 0.487284, "qid": "urgency" } ] }, { "id": "invoice_processing_000028", "workflow": "invoice_processing", "decisions": [ { "type": "score", "correct": true, "kl": 0.06444602759882834, "brier": 0.018597577858, "pred_score": 2.468484, "gold_score": 2.693333, "confidence": 0.642752, "qid": "discrepancy_severity" }, { "type": "choice", "correct": true, "kl": 0.05476500274058553, "brier": 0.035094850924611745, "confidence": 0.4837715162284838, "qid": "disposition" }, { "type": "noul", "correct": true, "kl": 0.0049857040605650795, "brier": 0.0007470658579999994, "confidence": 0.954006, "qid": "duplicate" }, { "type": "noul", "correct": true, "kl": 0.032477130673504516, "brier": 0.012221410482000001, "confidence": 0.865162, "qid": "matches_order" }, { "type": "score", "correct": true, "kl": 0.07346318240525525, "brier": 0.044415768253027205, "pred_score": 1.7784942215057784, "gold_score": 1.766667, "confidence": 0.4932135067864932, "qid": "urgency" } ] }, { "id": "invoice_processing_000029", "workflow": "invoice_processing", "decisions": [ { "type": "score", "correct": true, "kl": 0.031038449163881862, "brier": 0.011321359306000008, "pred_score": 2.727107, "gold_score": 2.616667, "confidence": 0.784709, "qid": "discrepancy_severity" }, { "type": "choice", "correct": true, "kl": 0.02483319987813026, "brier": 0.0034046271399999976, "confidence": 0.700978, "qid": "disposition" }, { "type": "noul", "correct": true, "kl": 0.0008507094836368934, "brier": 8.478624200000058e-05, "confidence": 0.976511, "qid": "duplicate" }, { "type": "noul", "correct": true, "kl": 0.0003164309478179191, "brier": 4.4443592000000976e-05, "confidence": 0.961953, "qid": "matches_order" }, { "type": "score", "correct": false, "kl": 0.07181652576512777, "brier": 0.057595506802, "pred_score": 1.396482, "gold_score": 1.6, "confidence": 0.55998, "qid": "urgency" } ] }, { "id": "invoice_processing_000030", "workflow": "invoice_processing", "decisions": [ { "type": "score", "correct": true, "kl": 0.11906252875078886, "brier": 0.03160211845499999, "pred_score": 0.49142900000000006, "gold_score": 0.106667, "confidence": 0.786234, "qid": "discrepancy_severity" }, { "type": "choice", "correct": false, "kl": 0.167233209615339, "brier": 0.123795302862, "confidence": 0.477728, "qid": "disposition" }, { "type": "noul", "correct": true, "kl": 0.06003642382605598, "brier": 0.054833472799999984, "confidence": 0.58442, "qid": "duplicate" }, { "type": "noul", "correct": true, "kl": 0.021818585643630498, "brier": 0.006912763361999991, "confidence": 0.891209, "qid": "matches_order" }, { "type": "score", "correct": true, "kl": 0.01708245377658113, "brier": 0.0035894126339999968, "pred_score": 1.218804, "gold_score": 1.173333, "confidence": 0.680873, "qid": "urgency" } ] }, { "id": "invoice_processing_000031", "workflow": "invoice_processing", "decisions": [ { "type": "score", "correct": true, "kl": 0.15720640604676234, "brier": 0.03346587546199998, "pred_score": 0.288777, "gold_score": 0.003333, "confidence": 0.843813, "qid": "discrepancy_severity" }, { "type": "choice", "correct": true, "kl": 0.1470722941914684, "brier": 0.05035219194199999, "confidence": 0.769779, "qid": "disposition" }, { "type": "noul", "correct": true, "kl": 0.010026960884156236, "brier": 0.001410973441999997, "confidence": 0.953439, "qid": "duplicate" }, { "type": "noul", "correct": true, "kl": 0.09477841200480533, "brier": 0.020314023047999986, "confidence": 0.895885, "qid": "matches_order" }, { "type": "score", "correct": false, "kl": 0.2382390373622636, "brier": 0.142066240758, "pred_score": 2.096201, "gold_score": 2.583333, "confidence": 0.477854, "qid": "urgency" } ] }, { "id": "invoice_processing_000032", "workflow": "invoice_processing", "decisions": [ { "type": "score", "correct": false, "kl": 0.07420518581964508, "brier": 0.050596591138000004, "pred_score": 0.912954, "gold_score": 1.186667, "confidence": 0.511112, "qid": "discrepancy_severity" }, { "type": "choice", "correct": true, "kl": 0.09159164215214022, "brier": 0.04983124595400003, "confidence": 0.5008250000000001, "qid": "disposition" }, { "type": "noul", "correct": true, "kl": 0.03805201595268866, "brier": 0.006897955968, "confidence": 0.927939, "qid": "duplicate" }, { "type": "noul", "correct": true, "kl": 0.028717039852435233, "brier": 0.019216232882000004, "confidence": 0.751979, "qid": "matches_order" }, { "type": "score", "correct": true, "kl": 0.05199979740671909, "brier": 0.024680535233999983, "pred_score": 1.266809, "gold_score": 1.083333, "confidence": 0.6510440000000001, "qid": "urgency" } ] }, { "id": "invoice_processing_000033", "workflow": "invoice_processing", "decisions": [ { "type": "score", "correct": false, "kl": 0.4629969392926891, "brier": 0.185837313205, "pred_score": 1.5950630000000001, "gold_score": 2.55, "confidence": 0.361899, "qid": "discrepancy_severity" }, { "type": "choice", "correct": true, "kl": 0.330464540124367, "brier": 0.19996817740199996, "confidence": 0.385481, "qid": "disposition" }, { "type": "noul", "correct": true, "kl": 0.005617446830666627, "brier": 0.000607679522000003, "confidence": 0.977431, "qid": "duplicate" }, { "type": "noul", "correct": true, "kl": 0.26437449684590336, "brier": 0.18105153125, "confidence": 0.649125, "qid": "matches_order" }, { "type": "score", "correct": true, "kl": 0.04045627148895561, "brier": 0.028786811832999998, "pred_score": 1.284191, "gold_score": 1.416667, "confidence": 0.659514, "qid": "urgency" } ] }, { "id": "invoice_processing_000034", "workflow": "invoice_processing", "decisions": [ { "type": "score", "correct": true, "kl": 0.12149331826153112, "brier": 0.030486217986000007, "pred_score": 0.340403, "gold_score": 0.038333, "confidence": 0.822085, "qid": "discrepancy_severity" }, { "type": "choice", "correct": true, "kl": 0.17178294237807684, "brier": 0.07681989589899997, "confidence": 0.681834, "qid": "disposition" }, { "type": "noul", "correct": true, "kl": 0.07492665967998541, "brier": 0.023690114449999983, "confidence": 0.867832, "qid": "duplicate" }, { "type": "noul", "correct": true, "kl": 0.08754985849304231, "brier": 0.026312180000000008, "confidence": 0.868633, "qid": "matches_order" }, { "type": "score", "correct": false, "kl": 0.19462748836089283, "brier": 0.14860403477399997, "pred_score": 2.184641, "gold_score": 2.593333, "confidence": 0.456401, "qid": "urgency" } ] }, { "id": "invoice_processing_000035", "workflow": "invoice_processing", "decisions": [ { "type": "score", "correct": true, "kl": 0.017556993323429207, "brier": 0.00791510173599999, "pred_score": 2.6755020000000003, "gold_score": 2.766667, "confidence": 0.713689, "qid": "discrepancy_severity" }, { "type": "choice", "correct": true, "kl": 0.017012832020047886, "brier": 0.005633021117999999, "confidence": 0.714277, "qid": "disposition" }, { "type": "noul", "correct": true, "kl": 0.00032928282327054567, "brier": 2.689377800000064e-05, "confidence": 0.980334, "qid": "duplicate" }, { "type": "noul", "correct": true, "kl": 0.004255090286703751, "brier": 0.0003917760319999972, "confidence": 0.980663, "qid": "matches_order" }, { "type": "score", "correct": true, "kl": 0.03641074985415663, "brier": 0.024400552646, "pred_score": 2.0903430000000003, "gold_score": 2.233333, "confidence": 0.524448, "qid": "urgency" } ] }, { "id": "invoice_processing_000036", "workflow": "invoice_processing", "decisions": [ { "type": "score", "correct": true, "kl": 0.05199659411303757, "brier": 0.009559235343999987, "pred_score": 0.199017, "gold_score": 0.043333, "confidence": 0.889094, "qid": "discrepancy_severity" }, { "type": "choice", "correct": true, "kl": 0.052592687610559755, "brier": 0.016302788022, "confidence": 0.830871, "qid": "disposition" }, { "type": "noul", "correct": true, "kl": 0.0062439110434576225, "brier": 0.0006862030580000024, "confidence": 0.96481, "qid": "duplicate" }, { "type": "noul", "correct": true, "kl": 0.034264755599111185, "brier": 0.0065842402580000015, "confidence": 0.925956, "qid": "matches_order" }, { "type": "score", "correct": false, "kl": 0.01728419609295638, "brier": 0.008529395881513805, "pred_score": 2.184194184194184, "gold_score": 2.32, "confidence": 0.4554764554764555, "qid": "urgency" } ] }, { "id": "invoice_processing_000037", "workflow": "invoice_processing", "decisions": [ { "type": "score", "correct": true, "kl": 0.10150532129407736, "brier": 0.026462592017999984, "pred_score": 0.317394, "gold_score": 0.051667, "confidence": 0.825049, "qid": "discrepancy_severity" }, { "type": "choice", "correct": true, "kl": 0.05275735126336167, "brier": 0.023097345570000004, "confidence": 0.75945, "qid": "disposition" }, { "type": "noul", "correct": true, "kl": 0.41274864911610193, "brier": 0.17089027219999997, "confidence": 0.948977, "qid": "duplicate" }, { "type": "noul", "correct": true, "kl": 0.05516240025597867, "brier": 0.015252708481999996, "confidence": 0.889338, "qid": "matches_order" }, { "type": "score", "correct": true, "kl": 0.029040476136832215, "brier": 0.0043284598034816985, "pred_score": 2.190890809109191, "gold_score": 2.316667, "confidence": 0.4697025302974697, "qid": "urgency" } ] }, { "id": "invoice_processing_000038", "workflow": "invoice_processing", "decisions": [ { "type": "score", "correct": true, "kl": 0.1324465137874041, "brier": 0.02936318975, "pred_score": 0.411059, "gold_score": 0.056667, "confidence": 0.808761, "qid": "discrepancy_severity" }, { "type": "choice", "correct": true, "kl": 0.1958915483012527, "brier": 0.10362733511199998, "confidence": 0.623103, "qid": "disposition" }, { "type": "noul", "correct": false, "kl": 0.22649762634275533, "brier": 0.19402961356800003, "confidence": 0.771472, "qid": "duplicate" }, { "type": "noul", "correct": true, "kl": 0.05189314143335188, "brier": 0.014904701858000007, "confidence": 0.887006, "qid": "matches_order" }, { "type": "score", "correct": true, "kl": 0.1796497302099292, "brier": 0.060664403716000015, "pred_score": 1.108378, "gold_score": 1.466667, "confidence": 0.74084, "qid": "urgency" } ] }, { "id": "invoice_processing_000039", "workflow": "invoice_processing", "decisions": [ { "type": "score", "correct": true, "kl": 0.02374116623767196, "brier": 0.006247556533999999, "pred_score": 2.2572330000000003, "gold_score": 2.393333, "confidence": 0.5249, "qid": "discrepancy_severity" }, { "type": "choice", "correct": true, "kl": 0.037435359205336825, "brier": 0.005713144728999996, "confidence": 0.637182, "qid": "disposition" }, { "type": "noul", "correct": true, "kl": 1.998554536371158e-05, "brier": 2.272712000000019e-06, "confidence": 0.971066, "qid": "duplicate" }, { "type": "noul", "correct": true, "kl": 0.0018822380507738119, "brier": 0.00021987044999999933, "confidence": 0.966182, "qid": "matches_order" }, { "type": "score", "correct": true, "kl": 0.007766847345805811, "brier": 0.0038677921959999976, "pred_score": 1.433432, "gold_score": 1.35, "confidence": 0.553621, "qid": "urgency" } ] }, { "id": "invoice_processing_000040", "workflow": "invoice_processing", "decisions": [ { "type": "score", "correct": true, "kl": 0.2492470585738913, "brier": 0.083701861784, "pred_score": 0.358489, "gold_score": 0.956667, "confidence": 0.833781, "qid": "discrepancy_severity" }, { "type": "choice", "correct": false, "kl": 0.3201161832544228, "brier": 0.18558981808599997, "confidence": 0.673901, "qid": "disposition" }, { "type": "noul", "correct": false, "kl": 0.22643114709419335, "brier": 0.211326706962, "confidence": 0.708392, "qid": "duplicate" }, { "type": "noul", "correct": true, "kl": 0.19052615246263868, "brier": 0.09617182245000003, "confidence": 0.909285, "qid": "matches_order" }, { "type": "score", "correct": false, "kl": 0.17183718701280398, "brier": 0.10613896405340946, "pred_score": 1.612944612944613, "gold_score": 1.25, "confidence": 0.4370494370494371, "qid": "urgency" } ] }, { "id": "invoice_processing_000041", "workflow": "invoice_processing", "decisions": [ { "type": "score", "correct": true, "kl": 0.29424234815583383, "brier": 0.10077974719000002, "pred_score": 0.38605100000000003, "gold_score": 1.036667, "confidence": 0.812473, "qid": "discrepancy_severity" }, { "type": "choice", "correct": false, "kl": 0.21933867457282855, "brier": 0.14830411636899998, "confidence": 0.518605, "qid": "disposition" }, { "type": "noul", "correct": true, "kl": 0.012659590869761816, "brier": 0.012186102727999996, "confidence": 0.568609, "qid": "duplicate" }, { "type": "noul", "correct": true, "kl": 0.282006104351304, "brier": 0.143188480082, "confidence": 0.917571, "qid": "matches_order" }, { "type": "score", "correct": true, "kl": 0.03929706221061971, "brier": 0.015728073850000003, "pred_score": 2.506069, "gold_score": 2.383333, "confidence": 0.564948, "qid": "urgency" } ] }, { "id": "invoice_processing_000042", "workflow": "invoice_processing", "decisions": [ { "type": "score", "correct": true, "kl": 0.10536777538150864, "brier": 0.021402553449999997, "pred_score": 0.408505, "gold_score": 0.103333, "confidence": 0.803586, "qid": "discrepancy_severity" }, { "type": "choice", "correct": true, "kl": 0.02128697684250797, "brier": 0.006515183191999997, "confidence": 0.576063, "qid": "disposition" }, { "type": "noul", "correct": true, "kl": 0.044960761425445944, "brier": 0.041056024351999974, "confidence": 0.689943, "qid": "duplicate" }, { "type": "noul", "correct": true, "kl": 0.08273657135862944, "brier": 0.022396591367999992, "confidence": 0.880845, "qid": "matches_order" }, { "type": "score", "correct": false, "kl": 0.0662043156905621, "brier": 0.050375094021034426, "pred_score": 2.1960721960721963, "gold_score": 2.433333, "confidence": 0.440929440929441, "qid": "urgency" } ] }, { "id": "invoice_processing_000043", "workflow": "invoice_processing", "decisions": [ { "type": "score", "correct": true, "kl": 0.5554758396533784, "brier": 0.25762245525, "pred_score": 0.33290000000000003, "gold_score": 1.303333, "confidence": 0.835882, "qid": "discrepancy_severity" }, { "type": "choice", "correct": false, "kl": 0.3726407180366785, "brier": 0.21250020715399998, "confidence": 0.639636, "qid": "disposition" }, { "type": "noul", "correct": false, "kl": 0.27850665676858655, "brier": 0.2520755606480001, "confidence": 0.741685, "qid": "duplicate" }, { "type": "noul", "correct": true, "kl": 0.021816864532470684, "brier": 0.005973900817999995, "confidence": 0.905347, "qid": "matches_order" }, { "type": "score", "correct": false, "kl": 0.07361474695812759, "brier": 0.03282510613399997, "pred_score": 2.160346, "gold_score": 2.433333, "confidence": 0.426712, "qid": "urgency" } ] }, { "id": "invoice_processing_000044", "workflow": "invoice_processing", "decisions": [ { "type": "score", "correct": true, "kl": 0.00960983001663454, "brier": 0.0039334585939999975, "pred_score": 2.846961, "gold_score": 2.79, "confidence": 0.874572, "qid": "discrepancy_severity" }, { "type": "choice", "correct": false, "kl": 0.21075523749441089, "brier": 0.15961229601000002, "confidence": 0.7301880000000001, "qid": "disposition" }, { "type": "noul", "correct": true, "kl": 0.0038793406634088125, "brier": 0.0004440200000000007, "confidence": 0.9651, "qid": "duplicate" }, { "type": "noul", "correct": true, "kl": 0.0011874740394844305, "brier": 0.00011497344799999963, "confidence": 0.977582, "qid": "matches_order" }, { "type": "score", "correct": true, "kl": 0.10747235746918278, "brier": 0.07212621327000002, "pred_score": 1.215727, "gold_score": 1.266667, "confidence": 0.652877, "qid": "urgency" } ] }, { "id": "invoice_processing_000045", "workflow": "invoice_processing", "decisions": [ { "type": "score", "correct": false, "kl": 0.36634428216416337, "brier": 0.22452639221700002, "pred_score": 1.587616, "gold_score": 0.726667, "confidence": 0.455224, "qid": "discrepancy_severity" }, { "type": "choice", "correct": false, "kl": 0.28097580692801766, "brier": 0.15748093857690618, "confidence": 0.3851746148253852, "qid": "disposition" }, { "type": "noul", "correct": true, "kl": 0.0013735574642265505, "brier": 0.00012000103199999874, "confidence": 0.974921, "qid": "duplicate" }, { "type": "noul", "correct": false, "kl": 0.173047990281417, "brier": 0.16893159380000003, "confidence": 0.63063, "qid": "matches_order" }, { "type": "score", "correct": true, "kl": 0.037776537836211016, "brier": 0.011454472142000008, "pred_score": 1.643046, "gold_score": 1.583333, "confidence": 0.487842, "qid": "urgency" } ] }, { "id": "invoice_processing_000046", "workflow": "invoice_processing", "decisions": [ { "type": "score", "correct": true, "kl": 0.005724608332234911, "brier": 0.0014896554659999963, "pred_score": 2.4699899999999997, "gold_score": 2.506667, "confidence": 0.623867, "qid": "discrepancy_severity" }, { "type": "choice", "correct": true, "kl": 0.04335962466711083, "brier": 0.03290395925799999, "confidence": 0.540106, "qid": "disposition" }, { "type": "noul", "correct": true, "kl": 0.01096488551383542, "brier": 0.0013442112500000047, "confidence": 0.975925, "qid": "duplicate" }, { "type": "noul", "correct": true, "kl": 0.00016974761216486094, "brier": 5.912193800000039e-05, "confidence": 0.905437, "qid": "matches_order" }, { "type": "score", "correct": false, "kl": 0.024020172509162496, "brier": 0.013903185945999996, "pred_score": 1.836945, "gold_score": 1.7, "confidence": 0.418996, "qid": "urgency" } ] }, { "id": "invoice_processing_000047", "workflow": "invoice_processing", "decisions": [ { "type": "score", "correct": true, "kl": 0.0419295602248906, "brier": 0.003439485918, "pred_score": 2.699399, "gold_score": 2.7, "confidence": 0.780238, "qid": "discrepancy_severity" }, { "type": "choice", "correct": true, "kl": 0.0992089545898551, "brier": 0.06400136868600001, "confidence": 0.680462, "qid": "disposition" }, { "type": "noul", "correct": true, "kl": 0.0026151688451321944, "brier": 0.00023078112800000057, "confidence": 0.980742, "qid": "duplicate" }, { "type": "noul", "correct": true, "kl": 0.01271083246133075, "brier": 0.0027500011220000023, "confidence": 0.953748, "qid": "matches_order" }, { "type": "score", "correct": true, "kl": 0.007751442345244373, "brier": 0.0014054021129999983, "pred_score": 1.260504, "gold_score": 1.2, "confidence": 0.643767, "qid": "urgency" } ] }, { "id": "invoice_processing_000048", "workflow": "invoice_processing", "decisions": [ { "type": "score", "correct": true, "kl": 0.09957770273565034, "brier": 0.025869998508, "pred_score": 1.151249, "gold_score": 1.486667, "confidence": 0.384157, "qid": "discrepancy_severity" }, { "type": "choice", "correct": false, "kl": 0.02838419045046291, "brier": 0.018092853403, "confidence": 0.344372, "qid": "disposition" }, { "type": "noul", "correct": true, "kl": 0.014783148551174918, "brier": 0.002565141937999999, "confidence": 0.940854, "qid": "duplicate" }, { "type": "noul", "correct": false, "kl": 0.06433933271741357, "brier": 0.061742385607999994, "confidence": 0.649035, "qid": "matches_order" }, { "type": "score", "correct": true, "kl": 0.030098005171151142, "brier": 0.0032067313097654164, "pred_score": 2.1369828630171366, "gold_score": 2.25, "confidence": 0.49330550669449325, "qid": "urgency" } ] }, { "id": "invoice_processing_000049", "workflow": "invoice_processing", "decisions": [ { "type": "score", "correct": true, "kl": 0.33738226848748654, "brier": 0.15072796228299998, "pred_score": 0.8756330000000001, "gold_score": 0.236667, "confidence": 0.5126120000000001, "qid": "discrepancy_severity" }, { "type": "choice", "correct": true, "kl": 0.32332140612465315, "brier": 0.22251181115312185, "confidence": 0.44672255327744675, "qid": "disposition" }, { "type": "noul", "correct": true, "kl": 0.002630340388798143, "brier": 0.00035820937800000094, "confidence": 0.95995, "qid": "duplicate" }, { "type": "noul", "correct": true, "kl": 0.09670517682820073, "brier": 0.05194251267200002, "confidence": 0.775511, "qid": "matches_order" }, { "type": "score", "correct": true, "kl": 0.056538616446003276, "brier": 0.019312047972000004, "pred_score": 2.129626, "gold_score": 2.35, "confidence": 0.494499, "qid": "urgency" } ] }, { "id": "invoice_processing_000050", "workflow": "invoice_processing", "decisions": [ { "type": "score", "correct": false, "kl": 0.05445017685836537, "brier": 0.041350754252533345, "pred_score": 1.0016689983310016, "gold_score": 1.303333, "confidence": 0.452949547050453, "qid": "discrepancy_severity" }, { "type": "choice", "correct": false, "kl": 0.17073286788520964, "brier": 0.10462230126783578, "confidence": 0.4042515957484043, "qid": "disposition" }, { "type": "noul", "correct": true, "kl": 0.0025586640140943043, "brier": 0.0003472139519999992, "confidence": 0.960157, "qid": "duplicate" }, { "type": "noul", "correct": true, "kl": 0.008222562297057777, "brier": 0.0070306081999999895, "confidence": 0.70929, "qid": "matches_order" }, { "type": "score", "correct": false, "kl": 0.0662553904906337, "brier": 0.05016014346799999, "pred_score": 2.1432160000000002, "gold_score": 2.333333, "confidence": 0.473696, "qid": "urgency" } ] }, { "id": "invoice_processing_000051", "workflow": "invoice_processing", "decisions": [ { "type": "score", "correct": true, "kl": 0.02741991089295817, "brier": 0.0029441030790000023, "pred_score": 2.728443, "gold_score": 2.84, "confidence": 0.849939, "qid": "discrepancy_severity" }, { "type": "choice", "correct": true, "kl": 0.14081128394142456, "brier": 0.03586063184700001, "confidence": 0.629964, "qid": "disposition" }, { "type": "noul", "correct": true, "kl": 0.0032086753141575674, "brier": 0.0003074704020000012, "confidence": 0.970934, "qid": "duplicate" }, { "type": "noul", "correct": true, "kl": 0.04472409025404102, "brier": 0.011339577607999998, "confidence": 0.901369, "qid": "matches_order" }, { "type": "score", "correct": false, "kl": 0.09064197907617699, "brier": 0.042242664453, "pred_score": 2.053869, "gold_score": 2.35, "confidence": 0.492647, "qid": "urgency" } ] }, { "id": "invoice_processing_000052", "workflow": "invoice_processing", "decisions": [ { "type": "score", "correct": true, "kl": 0.019991993047372474, "brier": 0.013878265621000006, "pred_score": 2.529242, "gold_score": 2.583333, "confidence": 0.593089, "qid": "discrepancy_severity" }, { "type": "choice", "correct": true, "kl": 0.040893293128201946, "brier": 0.0032441253979999946, "confidence": 0.614385, "qid": "disposition" }, { "type": "noul", "correct": true, "kl": 0.0007074099399540677, "brier": 6.316879999999954e-05, "confidence": 0.978953, "qid": "duplicate" }, { "type": "noul", "correct": true, "kl": 0.010122884237819645, "brier": 0.0017709580980000022, "confidence": 0.96309, "qid": "matches_order" }, { "type": "score", "correct": true, "kl": 0.17472057032609792, "brier": 0.06868349989857703, "pred_score": 1.7963257963257964, "gold_score": 2.253333, "confidence": 0.46103446103446105, "qid": "urgency" } ] }, { "id": "invoice_processing_000053", "workflow": "invoice_processing", "decisions": [ { "type": "score", "correct": true, "kl": 0.08983515982387795, "brier": 0.018595815458999992, "pred_score": 0.526479, "gold_score": 0.27, "confidence": 0.772325, "qid": "discrepancy_severity" }, { "type": "choice", "correct": true, "kl": 0.12711370879412887, "brier": 0.07887073599499997, "confidence": 0.602362, "qid": "disposition" }, { "type": "noul", "correct": false, "kl": 0.10307537549276785, "brier": 0.09863283465800005, "confidence": 0.662073, "qid": "duplicate" }, { "type": "noul", "correct": true, "kl": 0.057001648304710696, "brier": 0.01892407305800001, "confidence": 0.869394, "qid": "matches_order" }, { "type": "score", "correct": true, "kl": 0.04746212733751737, "brier": 0.008764827994, "pred_score": 1.201981, "gold_score": 1.036667, "confidence": 0.710713, "qid": "urgency" } ] }, { "id": "invoice_processing_000054", "workflow": "invoice_processing", "decisions": [ { "type": "score", "correct": true, "kl": 0.0478364491716674, "brier": 0.005046142985999999, "pred_score": 2.685297, "gold_score": 2.606667, "confidence": 0.712984, "qid": "discrepancy_severity" }, { "type": "choice", "correct": true, "kl": 0.12586096512668396, "brier": 0.019808133285999993, "confidence": 0.702506, "qid": "disposition" }, { "type": "noul", "correct": true, "kl": 0.00015268957061439527, "brier": 1.2954050000000093e-05, "confidence": 0.977455, "qid": "duplicate" }, { "type": "noul", "correct": true, "kl": 0.0712680554979982, "brier": 0.006414593378000006, "confidence": 0.989966, "qid": "matches_order" }, { "type": "score", "correct": true, "kl": 0.05127343052312701, "brier": 0.020159426979, "pred_score": 1.8241180000000001, "gold_score": 1.67, "confidence": 0.461444, "qid": "urgency" } ] }, { "id": "invoice_processing_000055", "workflow": "invoice_processing", "decisions": [ { "type": "score", "correct": true, "kl": 0.050397504993881515, "brier": 0.009184418246000001, "pred_score": 0.198438, "gold_score": 0.043333, "confidence": 0.88879, "qid": "discrepancy_severity" }, { "type": "choice", "correct": true, "kl": 0.06057773477942667, "brier": 0.017653330609999997, "confidence": 0.837857, "qid": "disposition" }, { "type": "noul", "correct": true, "kl": 0.023872952517097753, "brier": 0.0039615680720000036, "confidence": 0.938827, "qid": "duplicate" }, { "type": "noul", "correct": true, "kl": 0.03177758954772897, "brier": 0.005297704177999993, "confidence": 0.9352, "qid": "matches_order" }, { "type": "score", "correct": true, "kl": 0.06985475069667149, "brier": 0.03581408469400001, "pred_score": 1.993447, "gold_score": 1.816667, "confidence": 0.52918, "qid": "urgency" } ] }, { "id": "invoice_processing_000056", "workflow": "invoice_processing", "decisions": [ { "type": "score", "correct": false, "kl": 0.3245348992803332, "brier": 0.23399309203300003, "pred_score": 0.8610250000000002, "gold_score": 1.6, "confidence": 0.5305610000000001, "qid": "discrepancy_severity" }, { "type": "choice", "correct": false, "kl": 0.6713582837285539, "brier": 0.41789459907400006, "confidence": 0.546235, "qid": "disposition" }, { "type": "noul", "correct": true, "kl": 0.016349764437747424, "brier": 0.0020716197119999996, "confidence": 0.954483, "qid": "duplicate" }, { "type": "noul", "correct": true, "kl": 0.016792585855822925, "brier": 0.012968551250000009, "confidence": 0.763858, "qid": "matches_order" }, { "type": "score", "correct": false, "kl": 0.04863695069762105, "brier": 0.0392042778714172, "pred_score": 1.5231534768465234, "gold_score": 1.7, "confidence": 0.4498425501574499, "qid": "urgency" } ] }, { "id": "invoice_processing_000057", "workflow": "invoice_processing", "decisions": [ { "type": "score", "correct": true, "kl": 0.09466403554485413, "brier": 0.014509875125000002, "pred_score": 0.20808, "gold_score": 0.216667, "confidence": 0.887399, "qid": "discrepancy_severity" }, { "type": "choice", "correct": true, "kl": 0.009607197756382375, "brier": 0.0013903998340000022, "confidence": 0.846164, "qid": "disposition" }, { "type": "noul", "correct": true, "kl": 0.004920563138523088, "brier": 0.0005887252979999993, "confidence": 0.962843, "qid": "duplicate" }, { "type": "noul", "correct": true, "kl": 0.014245508654487076, "brier": 0.003324222721999996, "confidence": 0.922564, "qid": "matches_order" }, { "type": "score", "correct": true, "kl": 0.08871994350041046, "brier": 0.06190583914000001, "pred_score": 1.692982, "gold_score": 1.763333, "confidence": 0.427474, "qid": "urgency" } ] }, { "id": "invoice_processing_000058", "workflow": "invoice_processing", "decisions": [ { "type": "score", "correct": true, "kl": 0.016321452814153238, "brier": 0.003992445339000001, "pred_score": 2.780873, "gold_score": 2.76, "confidence": 0.840265, "qid": "discrepancy_severity" }, { "type": "choice", "correct": true, "kl": 0.048128835101839657, "brier": 0.006286172509999999, "confidence": 0.699249, "qid": "disposition" }, { "type": "noul", "correct": true, "kl": 7.428039888669096e-05, "brier": 6.4296980000001575e-06, "confidence": 0.97846, "qid": "duplicate" }, { "type": "noul", "correct": true, "kl": 0.01158463701702396, "brier": 0.0015016296020000006, "confidence": 0.955932, "qid": "matches_order" }, { "type": "score", "correct": true, "kl": 0.21780623679285321, "brier": 0.071129954782, "pred_score": 1.402614, "gold_score": 1.036667, "confidence": 0.501507, "qid": "urgency" } ] }, { "id": "invoice_processing_000059", "workflow": "invoice_processing", "decisions": [ { "type": "score", "correct": true, "kl": 0.09482563399064871, "brier": 0.029679436369999996, "pred_score": 2.327956, "gold_score": 2.616667, "confidence": 0.60266, "qid": "discrepancy_severity" }, { "type": "choice", "correct": false, "kl": 0.04535049738233529, "brier": 0.022925174743350343, "confidence": 0.4733465266534733, "qid": "disposition" }, { "type": "noul", "correct": true, "kl": 0.001088441900866129, "brier": 0.00023440455199999995, "confidence": 0.939174, "qid": "duplicate" }, { "type": "noul", "correct": true, "kl": 0.04141272348103184, "brier": 0.023064365088000006, "confidence": 0.792612, "qid": "matches_order" }, { "type": "score", "correct": false, "kl": 0.005253733460677645, "brier": 0.003962003525644343, "pred_score": 2.2874887125112875, "gold_score": 2.353333, "confidence": 0.4708945291054709, "qid": "urgency" } ] }, { "id": "invoice_processing_000060", "workflow": "invoice_processing", "decisions": [ { "type": "score", "correct": false, "kl": 0.17984853494517783, "brier": 0.08753113539399998, "pred_score": 1.608999, "gold_score": 1.473333, "confidence": 0.317306, "qid": "discrepancy_severity" }, { "type": "choice", "correct": false, "kl": 0.19520061571502584, "brier": 0.08783535745105986, "confidence": 0.38096638096638097, "qid": "disposition" }, { "type": "noul", "correct": true, "kl": 3.0547342558716003e-06, "brier": 2.422080000000112e-07, "confidence": 0.979652, "qid": "duplicate" }, { "type": "noul", "correct": true, "kl": 0.002646495543515426, "brier": 0.002406653442, "confidence": 0.638644, "qid": "matches_order" }, { "type": "score", "correct": true, "kl": 0.22074122878620128, "brier": 0.12205805322200002, "pred_score": 1.329025, "gold_score": 1.033333, "confidence": 0.625351, "qid": "urgency" } ] }, { "id": "invoice_processing_000061", "workflow": "invoice_processing", "decisions": [ { "type": "score", "correct": true, "kl": 0.12825865439838496, "brier": 0.04381002766910686, "pred_score": 2.0825489174510827, "gold_score": 2.516667, "confidence": 0.4110545889454111, "qid": "discrepancy_severity" }, { "type": "choice", "correct": true, "kl": 0.009294262750202025, "brier": 0.005696905093999998, "confidence": 0.586404, "qid": "disposition" }, { "type": "noul", "correct": true, "kl": 0.00047360030544048824, "brier": 7.390848199999935e-05, "confidence": 0.957254, "qid": "duplicate" }, { "type": "noul", "correct": true, "kl": 0.14843137524862693, "brier": 0.083267907872, "confidence": 0.745956, "qid": "matches_order" }, { "type": "score", "correct": true, "kl": 0.003920074930845684, "brier": 0.0014974172879999971, "pred_score": 2.092657, "gold_score": 2.15, "confidence": 0.517168, "qid": "urgency" } ] }, { "id": "invoice_processing_000062", "workflow": "invoice_processing", "decisions": [ { "type": "score", "correct": true, "kl": 0.14008744931599096, "brier": 0.037451203552999995, "pred_score": 0.48716400000000004, "gold_score": 0.083333, "confidence": 0.77535, "qid": "discrepancy_severity" }, { "type": "choice", "correct": true, "kl": 0.03622491353177431, "brier": 0.013817771141999996, "confidence": 0.589554, "qid": "disposition" }, { "type": "noul", "correct": true, "kl": 0.04427646105380813, "brier": 0.03474511604999998, "confidence": 0.771805, "qid": "duplicate" }, { "type": "noul", "correct": true, "kl": 0.054525051609323326, "brier": 0.018685924561999988, "confidence": 0.866674, "qid": "matches_order" }, { "type": "score", "correct": true, "kl": 0.1691530725509966, "brier": 0.045228820650000015, "pred_score": 1.309809, "gold_score": 0.946667, "confidence": 0.614335, "qid": "urgency" } ] }, { "id": "invoice_processing_000063", "workflow": "invoice_processing", "decisions": [ { "type": "score", "correct": true, "kl": 0.1814298296513205, "brier": 0.068371098502, "pred_score": 0.250685, "gold_score": 0.603333, "confidence": 0.872043, "qid": "discrepancy_severity" }, { "type": "choice", "correct": true, "kl": 0.17548399818225519, "brier": 0.058123644306000026, "confidence": 0.840438, "qid": "disposition" }, { "type": "noul", "correct": true, "kl": 0.044389614478835576, "brier": 0.007715801087999992, "confidence": 0.927888, "qid": "duplicate" }, { "type": "noul", "correct": true, "kl": 0.1544331992488608, "brier": 0.07497353644999999, "confidence": 0.910282, "qid": "matches_order" }, { "type": "score", "correct": true, "kl": 0.09233595620205179, "brier": 0.03860781344999999, "pred_score": 1.307945, "gold_score": 1.056667, "confidence": 0.646469, "qid": "urgency" } ] }, { "id": "invoice_processing_000064", "workflow": "invoice_processing", "decisions": [ { "type": "score", "correct": true, "kl": 0.11431324604623389, "brier": 0.026007789400000026, "pred_score": 0.36261499999999997, "gold_score": 0.043333, "confidence": 0.829853, "qid": "discrepancy_severity" }, { "type": "choice", "correct": true, "kl": 0.06334013225002762, "brier": 0.015064432168, "confidence": 0.669112, "qid": "disposition" }, { "type": "noul", "correct": true, "kl": 0.00241539958644367, "brier": 0.0018713738419999972, "confidence": 0.747256, "qid": "duplicate" }, { "type": "noul", "correct": true, "kl": 0.06926838181315402, "brier": 0.013752447857999995, "confidence": 0.91041, "qid": "matches_order" }, { "type": "score", "correct": true, "kl": 0.08528971049511559, "brier": 0.036892088581999986, "pred_score": 2.036208, "gold_score": 1.833333, "confidence": 0.584774, "qid": "urgency" } ] }, { "id": "invoice_processing_000065", "workflow": "invoice_processing", "decisions": [ { "type": "score", "correct": true, "kl": 0.3206021327178738, "brier": 0.19104761953400004, "pred_score": 0.955715, "gold_score": 0.256667, "confidence": 0.468678, "qid": "discrepancy_severity" }, { "type": "choice", "correct": true, "kl": 0.3826154713864619, "brier": 0.260203562828, "confidence": 0.439793, "qid": "disposition" }, { "type": "noul", "correct": true, "kl": 0.0061971075176303365, "brier": 0.0007775201779999995, "confidence": 0.960283, "qid": "duplicate" }, { "type": "noul", "correct": true, "kl": 0.07626190875057762, "brier": 0.043760652800000015, "confidence": 0.768747, "qid": "matches_order" }, { "type": "score", "correct": false, "kl": 0.06887565205806906, "brier": 0.056573849821999984, "pred_score": 2.2085820000000003, "gold_score": 2.423333, "confidence": 0.520398, "qid": "urgency" } ] }, { "id": "invoice_processing_000066", "workflow": "invoice_processing", "decisions": [ { "type": "score", "correct": true, "kl": 0.055789969663534306, "brier": 0.02390926966835192, "pred_score": 1.0169369830630168, "gold_score": 1.046667, "confidence": 0.44853255146744847, "qid": "discrepancy_severity" }, { "type": "choice", "correct": true, "kl": 0.11233511259452461, "brier": 0.060358940334999986, "confidence": 0.397774, "qid": "disposition" }, { "type": "noul", "correct": true, "kl": 0.0029568763419243627, "brier": 0.00040897999999999954, "confidence": 0.959033, "qid": "duplicate" }, { "type": "noul", "correct": true, "kl": 0.017600472024035058, "brier": 0.015234200351999998, "confidence": 0.710609, "qid": "matches_order" }, { "type": "score", "correct": true, "kl": 0.0077216590338482285, "brier": 0.0029761095380000007, "pred_score": 2.161686, "gold_score": 2.246667, "confidence": 0.461894, "qid": "urgency" } ] }, { "id": "invoice_processing_000067", "workflow": "invoice_processing", "decisions": [ { "type": "score", "correct": true, "kl": 0.05553522489785269, "brier": 0.011748556148000001, "pred_score": 0.234147, "gold_score": 0.06, "confidence": 0.868824, "qid": "discrepancy_severity" }, { "type": "choice", "correct": true, "kl": 0.07117283964476909, "brier": 0.02256838465399999, "confidence": 0.812534, "qid": "disposition" }, { "type": "noul", "correct": true, "kl": 0.012223244382223601, "brier": 0.0018152722580000009, "confidence": 0.949873, "qid": "duplicate" }, { "type": "noul", "correct": true, "kl": 0.028384266620506977, "brier": 0.006059683871999998, "confidence": 0.921623, "qid": "matches_order" }, { "type": "score", "correct": true, "kl": 0.08146604892043327, "brier": 0.031649690067, "pred_score": 2.113319, "gold_score": 2.393333, "confidence": 0.477015, "qid": "urgency" } ] }, { "id": "invoice_processing_000068", "workflow": "invoice_processing", "decisions": [ { "type": "score", "correct": true, "kl": 0.004917089212735015, "brier": 0.0018039813249999992, "pred_score": 1.8295000000000001, "gold_score": 1.849583, "confidence": 0.661571, "qid": "discrepancy_severity" }, { "type": "choice", "correct": true, "kl": 0.024277111026041957, "brier": 0.018884683492999984, "confidence": 0.513441, "qid": "disposition" }, { "type": "noul", "correct": true, "kl": 0.0073683886812964255, "brier": 0.0014148008179999988, "confidence": 0.939736, "qid": "duplicate" }, { "type": "noul", "correct": true, "kl": 0.018088250260832328, "brier": 0.009795520512000007, "confidence": 0.813349, "qid": "matches_order" }, { "type": "score", "correct": false, "kl": 0.08807177083430638, "brier": 0.071572578074, "pred_score": 1.516782, "gold_score": 1.7, "confidence": 0.45046, "qid": "urgency" } ] }, { "id": "invoice_processing_000069", "workflow": "invoice_processing", "decisions": [ { "type": "score", "correct": true, "kl": 0.09328938652990522, "brier": 0.019100322791999996, "pred_score": 0.24102600000000002, "gold_score": 0.023333, "confidence": 0.867059, "qid": "discrepancy_severity" }, { "type": "choice", "correct": true, "kl": 0.07107823491687934, "brier": 0.02235400955999999, "confidence": 0.816381, "qid": "disposition" }, { "type": "noul", "correct": true, "kl": 0.01834299661572572, "brier": 0.0024276511999999986, "confidence": 0.951827, "qid": "duplicate" }, { "type": "noul", "correct": true, "kl": 0.0563436983461227, "brier": 0.011172733127999997, "confidence": 0.915258, "qid": "matches_order" }, { "type": "score", "correct": true, "kl": 0.04820962223580616, "brier": 0.008888111885000003, "pred_score": 2.299838, "gold_score": 2.45, "confidence": 0.458607, "qid": "urgency" } ] }, { "id": "invoice_processing_000070", "workflow": "invoice_processing", "decisions": [ { "type": "score", "correct": true, "kl": 0.04822281959428287, "brier": 0.02393259212200001, "pred_score": 2.727553, "gold_score": 2.866667, "confidence": 0.766694, "qid": "discrepancy_severity" }, { "type": "choice", "correct": true, "kl": 0.010847865792683088, "brier": 0.002031937201, "confidence": 0.665903, "qid": "disposition" }, { "type": "noul", "correct": true, "kl": 0.002437933600630925, "brier": 0.0001483675379999997, "confidence": 0.981387, "qid": "duplicate" }, { "type": "noul", "correct": true, "kl": 0.0004408777843407565, "brier": 3.538087200000035e-05, "confidence": 0.980873, "qid": "matches_order" }, { "type": "score", "correct": true, "kl": 0.004408500880538817, "brier": 0.000933098158999999, "pred_score": 1.260165, "gold_score": 1.3, "confidence": 0.632883, "qid": "urgency" } ] }, { "id": "invoice_processing_000071", "workflow": "invoice_processing", "decisions": [ { "type": "score", "correct": false, "kl": 0.2067085577633481, "brier": 0.12179848556600002, "pred_score": 1.157451, "gold_score": 1.583333, "confidence": 0.410604, "qid": "discrepancy_severity" }, { "type": "choice", "correct": false, "kl": 0.12026532455985975, "brier": 0.069477957482, "confidence": 0.40794, "qid": "disposition" }, { "type": "noul", "correct": true, "kl": 0.0043444438533243295, "brier": 0.00037248121799999905, "confidence": 0.97302, "qid": "duplicate" }, { "type": "noul", "correct": true, "kl": 0.08640302606779009, "brier": 0.06693335544199999, "confidence": 0.667061, "qid": "matches_order" }, { "type": "score", "correct": false, "kl": 0.02121583395931011, "brier": 0.013306445339786364, "pred_score": 1.4764585235414767, "gold_score": 1.536667, "confidence": 0.47490752509247497, "qid": "urgency" } ] }, { "id": "invoice_processing_000072", "workflow": "invoice_processing", "decisions": [ { "type": "score", "correct": true, "kl": 0.15486759008255963, "brier": 0.10909989317199997, "pred_score": 2.3246, "gold_score": 2.653333, "confidence": 0.52009, "qid": "discrepancy_severity" }, { "type": "choice", "correct": true, "kl": 0.07124957270348951, "brier": 0.042226004098, "confidence": 0.530955, "qid": "disposition" }, { "type": "noul", "correct": true, "kl": 0.002095787078214905, "brier": 0.00015607911200000014, "confidence": 0.977833, "qid": "duplicate" }, { "type": "noul", "correct": true, "kl": 0.012772337233933423, "brier": 0.004465881032, "confidence": 0.886079, "qid": "matches_order" }, { "type": "score", "correct": true, "kl": 0.02820757119832134, "brier": 0.005181318613000004, "pred_score": 1.332876, "gold_score": 1.233333, "confidence": 0.597074, "qid": "urgency" } ] }, { "id": "invoice_processing_000073", "workflow": "invoice_processing", "decisions": [ { "type": "score", "correct": true, "kl": 0.02145969978310138, "brier": 0.008211647411999992, "pred_score": 2.784033, "gold_score": 2.83, "confidence": 0.809092, "qid": "discrepancy_severity" }, { "type": "choice", "correct": true, "kl": 0.036655118136967695, "brier": 0.003393954915, "confidence": 0.716318, "qid": "disposition" }, { "type": "noul", "correct": true, "kl": 0.0023767128461912226, "brier": 0.0001862450000000009, "confidence": 0.982983, "qid": "duplicate" }, { "type": "noul", "correct": true, "kl": 0.005006054449951508, "brier": 0.0002917078580000001, "confidence": 0.988744, "qid": "matches_order" }, { "type": "score", "correct": false, "kl": 0.27189898415128094, "brier": 0.15346808858199998, "pred_score": 1.3345630000000002, "gold_score": 1.85, "confidence": 0.517035, "qid": "urgency" } ] }, { "id": "invoice_processing_000074", "workflow": "invoice_processing", "decisions": [ { "type": "score", "correct": true, "kl": 0.33670114120660927, "brier": 0.19372908425479343, "pred_score": 1.5310175310175314, "gold_score": 0.678333, "confidence": 0.40674040674040673, "qid": "discrepancy_severity" }, { "type": "choice", "correct": true, "kl": 0.13925235867105673, "brier": 0.07622079270200001, "confidence": 0.405466, "qid": "disposition" }, { "type": "noul", "correct": true, "kl": 0.0011828843844049414, "brier": 0.00010093363199999997, "confidence": 0.980437, "qid": "duplicate" }, { "type": "noul", "correct": true, "kl": 0.07642974099990665, "brier": 0.073640473992, "confidence": 0.518114, "qid": "matches_order" }, { "type": "score", "correct": true, "kl": 0.13352698236476004, "brier": 0.04327140886100001, "pred_score": 1.9640909999999998, "gold_score": 2.3, "confidence": 0.509335, "qid": "urgency" } ] }, { "id": "invoice_processing_000075", "workflow": "invoice_processing", "decisions": [ { "type": "score", "correct": true, "kl": 0.07063409040015126, "brier": 0.012447742024000018, "pred_score": 0.236242, "gold_score": 0.040333, "confidence": 0.873539, "qid": "discrepancy_severity" }, { "type": "choice", "correct": true, "kl": 0.05642955347627267, "brier": 0.015110580181999985, "confidence": 0.854809, "qid": "disposition" }, { "type": "noul", "correct": true, "kl": 0.01024243840481263, "brier": 0.0009220076820000006, "confidence": 0.968529, "qid": "duplicate" }, { "type": "noul", "correct": true, "kl": 0.0541223324547067, "brier": 0.012792321152000009, "confidence": 0.903357, "qid": "matches_order" }, { "type": "score", "correct": true, "kl": 0.058197561117746696, "brier": 0.010186093165000007, "pred_score": 1.491925, "gold_score": 1.433333, "confidence": 0.492442, "qid": "urgency" } ] }, { "id": "invoice_processing_000076", "workflow": "invoice_processing", "decisions": [ { "type": "score", "correct": false, "kl": 0.038508730114505146, "brier": 0.029305466004511855, "pred_score": 2.378001378001378, "gold_score": 2.54, "confidence": 0.48851448851448853, "qid": "discrepancy_severity" }, { "type": "choice", "correct": true, "kl": 0.01710577646559968, "brier": 0.005675633270000006, "confidence": 0.669386, "qid": "disposition" }, { "type": "noul", "correct": true, "kl": 0.0002703527038668404, "brier": 3.211209799999986e-05, "confidence": 0.970674, "qid": "duplicate" }, { "type": "noul", "correct": true, "kl": 0.05353991556029193, "brier": 0.009864224881999999, "confidence": 0.970229, "qid": "matches_order" }, { "type": "score", "correct": true, "kl": 0.05381062067739012, "brier": 0.029915146095000008, "pred_score": 1.817661, "gold_score": 1.913333, "confidence": 0.520253, "qid": "urgency" } ] }, { "id": "invoice_processing_000077", "workflow": "invoice_processing", "decisions": [ { "type": "score", "correct": true, "kl": 0.020463077635168553, "brier": 0.003647064489000001, "pred_score": 2.608997, "gold_score": 2.576667, "confidence": 0.66105, "qid": "discrepancy_severity" }, { "type": "choice", "correct": true, "kl": 0.14129924479409797, "brier": 0.088008195875, "confidence": 0.707306, "qid": "disposition" }, { "type": "noul", "correct": true, "kl": 0.0036792491235798693, "brier": 0.0004267288980000011, "confidence": 0.974607, "qid": "duplicate" }, { "type": "noul", "correct": true, "kl": 0.14438760506647624, "brier": 0.022636387538, "confidence": 0.983054, "qid": "matches_order" }, { "type": "score", "correct": true, "kl": 0.02382964318693383, "brier": 0.015673924402999993, "pred_score": 1.831947, "gold_score": 1.75, "confidence": 0.46031, "qid": "urgency" } ] }, { "id": "invoice_processing_000078", "workflow": "invoice_processing", "decisions": [ { "type": "score", "correct": true, "kl": 0.1881491953625687, "brier": 0.13561089867000004, "pred_score": 2.471647, "gold_score": 2.8, "confidence": 0.520274, "qid": "discrepancy_severity" }, { "type": "choice", "correct": true, "kl": 0.003620604838070777, "brier": 0.0005974054439999997, "confidence": 0.708688, "qid": "disposition" }, { "type": "noul", "correct": true, "kl": 0.000549492859632919, "brier": 7.594048800000002e-05, "confidence": 0.966162, "qid": "duplicate" }, { "type": "noul", "correct": true, "kl": 0.0001832516184000683, "brier": 1.0895111999999759e-05, "confidence": 0.985667, "qid": "matches_order" }, { "type": "score", "correct": false, "kl": 0.1609579922668587, "brier": 0.11511607608598773, "pred_score": 1.8822281177718825, "gold_score": 1.6, "confidence": 0.4901415098584902, "qid": "urgency" } ] }, { "id": "invoice_processing_000079", "workflow": "invoice_processing", "decisions": [ { "type": "score", "correct": true, "kl": 0.07552357305325862, "brier": 0.007250486299, "pred_score": 2.668158, "gold_score": 2.84, "confidence": 0.827848, "qid": "discrepancy_severity" }, { "type": "choice", "correct": true, "kl": 0.12698577394058586, "brier": 0.03351900672199999, "confidence": 0.610001, "qid": "disposition" }, { "type": "noul", "correct": true, "kl": 0.0003276020621224805, "brier": 3.4661138000000234e-05, "confidence": 0.974163, "qid": "duplicate" }, { "type": "noul", "correct": true, "kl": 0.056690079804975806, "brier": 0.019694773512000007, "confidence": 0.864099, "qid": "matches_order" }, { "type": "score", "correct": false, "kl": 0.08020275118733647, "brier": 0.04485544313399999, "pred_score": 2.036874, "gold_score": 2.316667, "confidence": 0.509833, "qid": "urgency" } ] }, { "id": "invoice_processing_000080", "workflow": "invoice_processing", "decisions": [ { "type": "score", "correct": true, "kl": 0.03987603149450766, "brier": 0.003919525810000001, "pred_score": 2.487337, "gold_score": 2.61, "confidence": 0.651534, "qid": "discrepancy_severity" }, { "type": "choice", "correct": false, "kl": 0.17427089713123095, "brier": 0.10318530547399998, "confidence": 0.638929, "qid": "disposition" }, { "type": "noul", "correct": true, "kl": 0.005996029126649533, "brier": 0.0007469885519999982, "confidence": 0.960674, "qid": "duplicate" }, { "type": "noul", "correct": true, "kl": 0.009693712922415207, "brier": 0.0028835936819999984, "confidence": 0.905362, "qid": "matches_order" }, { "type": "score", "correct": true, "kl": 0.03383615144061624, "brier": 0.023056517874000027, "pred_score": 1.992609, "gold_score": 2.016667, "confidence": 0.656074, "qid": "urgency" } ] }, { "id": "invoice_processing_000081", "workflow": "invoice_processing", "decisions": [ { "type": "score", "correct": true, "kl": 0.08075257847139669, "brier": 0.01654329978999998, "pred_score": 0.317376, "gold_score": 0.076667, "confidence": 0.835331, "qid": "discrepancy_severity" }, { "type": "choice", "correct": true, "kl": 0.012540711253914646, "brier": 0.0030528162340000015, "confidence": 0.785998, "qid": "disposition" }, { "type": "noul", "correct": true, "kl": 0.0001321612473298146, "brier": 2.7855648000000504e-05, "confidence": 0.942935, "qid": "duplicate" }, { "type": "noul", "correct": true, "kl": 0.04748855520226945, "brier": 0.015522162817999998, "confidence": 0.875236, "qid": "matches_order" }, { "type": "score", "correct": false, "kl": 0.09902983483700824, "brier": 0.07851837249900001, "pred_score": 1.514438, "gold_score": 1.74, "confidence": 0.45998, "qid": "urgency" } ] }, { "id": "invoice_processing_000082", "workflow": "invoice_processing", "decisions": [ { "type": "score", "correct": true, "kl": 0.08992554319455306, "brier": 0.009209478877999996, "pred_score": 0.31826299999999996, "gold_score": 0.16, "confidence": 0.838852, "qid": "discrepancy_severity" }, { "type": "choice", "correct": true, "kl": 0.02168683198946925, "brier": 0.006796237921999999, "confidence": 0.770381, "qid": "disposition" }, { "type": "noul", "correct": true, "kl": 0.023785084622103277, "brier": 0.004778508799999992, "confidence": 0.927787, "qid": "duplicate" }, { "type": "noul", "correct": true, "kl": 0.0356518558004288, "brier": 0.012225162978, "confidence": 0.87515, "qid": "matches_order" }, { "type": "score", "correct": true, "kl": 0.005347710759447244, "brier": 0.0030697028359999994, "pred_score": 1.147978, "gold_score": 1.123333, "confidence": 0.72114, "qid": "urgency" } ] }, { "id": "invoice_processing_000083", "workflow": "invoice_processing", "decisions": [ { "type": "score", "correct": true, "kl": 0.07707612512729048, "brier": 0.05732989160700005, "pred_score": 2.5708059999999997, "gold_score": 2.78, "confidence": 0.619234, "qid": "discrepancy_severity" }, { "type": "choice", "correct": true, "kl": 0.009605537935239671, "brier": 0.002694833629000005, "confidence": 0.723048, "qid": "disposition" }, { "type": "noul", "correct": true, "kl": 0.0004853741492836898, "brier": 6.321376800000019e-05, "confidence": 0.964378, "qid": "duplicate" }, { "type": "noul", "correct": true, "kl": 6.112351138013892e-05, "brier": 4.554161999999947e-06, "confidence": 0.981509, "qid": "matches_order" }, { "type": "score", "correct": true, "kl": 0.012769189879432784, "brier": 0.0025451069959999974, "pred_score": 1.9582650000000001, "gold_score": 1.87, "confidence": 0.582705, "qid": "urgency" } ] }, { "id": "invoice_processing_000084", "workflow": "invoice_processing", "decisions": [ { "type": "score", "correct": true, "kl": 0.15667026046745663, "brier": 0.10941590666599996, "pred_score": 2.562244, "gold_score": 2.81, "confidence": 0.629357, "qid": "discrepancy_severity" }, { "type": "choice", "correct": false, "kl": 0.05514400108320354, "brier": 0.03956311799400002, "confidence": 0.516047, "qid": "disposition" }, { "type": "noul", "correct": true, "kl": 0.004249640083333211, "brier": 0.0006201537619999996, "confidence": 0.955724, "qid": "duplicate" }, { "type": "noul", "correct": true, "kl": 0.0015888581363535399, "brier": 0.0002895380479999984, "confidence": 0.947968, "qid": "matches_order" }, { "type": "score", "correct": true, "kl": 0.2193770262202986, "brier": 0.05862745208999998, "pred_score": 1.3166639999999998, "gold_score": 1.683333, "confidence": 0.56073, "qid": "urgency" } ] }, { "id": "invoice_processing_000085", "workflow": "invoice_processing", "decisions": [ { "type": "score", "correct": true, "kl": 0.16765164041440667, "brier": 0.12833684371799997, "pred_score": 2.433659, "gold_score": 2.766667, "confidence": 0.512236, "qid": "discrepancy_severity" }, { "type": "choice", "correct": true, "kl": 0.13352590101808753, "brier": 0.10226685115800001, "confidence": 0.696435, "qid": "disposition" }, { "type": "noul", "correct": true, "kl": 0.0007230029195365245, "brier": 6.714087199999962e-05, "confidence": 0.974206, "qid": "duplicate" }, { "type": "noul", "correct": true, "kl": 0.006546951938985791, "brier": 0.0010274044500000023, "confidence": 0.965998, "qid": "matches_order" }, { "type": "score", "correct": true, "kl": 0.07381251392672937, "brier": 0.05425890514083406, "pred_score": 1.8140631859368141, "gold_score": 1.883333, "confidence": 0.4646105353894646, "qid": "urgency" } ] }, { "id": "invoice_processing_000086", "workflow": "invoice_processing", "decisions": [ { "type": "score", "correct": true, "kl": 0.09385386382506468, "brier": 0.014343718533999999, "pred_score": 0.23756100000000002, "gold_score": 0.03, "confidence": 0.8721, "qid": "discrepancy_severity" }, { "type": "choice", "correct": true, "kl": 0.03882884494069552, "brier": 0.00706658037799999, "confidence": 0.84055, "qid": "disposition" }, { "type": "noul", "correct": true, "kl": 0.008497684814762325, "brier": 0.001146630271999998, "confidence": 0.956056, "qid": "duplicate" }, { "type": "noul", "correct": true, "kl": 0.04674734947249519, "brier": 0.009368067199999993, "confidence": 0.918227, "qid": "matches_order" }, { "type": "score", "correct": true, "kl": 0.24649647456467114, "brier": 0.11479870271399999, "pred_score": 1.658771, "gold_score": 1.683333, "confidence": 0.41738, "qid": "urgency" } ] }, { "id": "invoice_processing_000087", "workflow": "invoice_processing", "decisions": [ { "type": "score", "correct": true, "kl": 0.07009686410057192, "brier": 0.02469300278600001, "pred_score": 2.555962, "gold_score": 2.75, "confidence": 0.698461, "qid": "discrepancy_severity" }, { "type": "choice", "correct": false, "kl": 0.16449483396453532, "brier": 0.09889894510299999, "confidence": 0.604984, "qid": "disposition" }, { "type": "noul", "correct": true, "kl": 0.0002111044931334999, "brier": 1.5323647999999957e-05, "confidence": 0.980565, "qid": "duplicate" }, { "type": "noul", "correct": true, "kl": 0.004178174650136979, "brier": 0.0010086336979999973, "confidence": 0.927543, "qid": "matches_order" }, { "type": "score", "correct": true, "kl": 0.13999497658105342, "brier": 0.07785041533800001, "pred_score": 1.5738120000000002, "gold_score": 1.883333, "confidence": 0.449062, "qid": "urgency" } ] }, { "id": "invoice_processing_000088", "workflow": "invoice_processing", "decisions": [ { "type": "score", "correct": false, "kl": 0.08063738286157578, "brier": 0.05825450000599999, "pred_score": 0.923988, "gold_score": 1.31, "confidence": 0.476345, "qid": "discrepancy_severity" }, { "type": "choice", "correct": false, "kl": 0.1653062767833269, "brier": 0.09772960395747007, "confidence": 0.455013544986455, "qid": "disposition" }, { "type": "noul", "correct": true, "kl": 0.00558444902551385, "brier": 0.0010937164499999997, "confidence": 0.939948, "qid": "duplicate" }, { "type": "noul", "correct": true, "kl": 0.020354250983780034, "brier": 0.014480275842000009, "confidence": 0.795089, "qid": "matches_order" }, { "type": "score", "correct": false, "kl": 0.12198972906185421, "brier": 0.09576638618000002, "pred_score": 2.286574, "gold_score": 2.566667, "confidence": 0.501819, "qid": "urgency" } ] }, { "id": "invoice_processing_000089", "workflow": "invoice_processing", "decisions": [ { "type": "score", "correct": true, "kl": 0.05687786986835314, "brier": 0.008660657815999994, "pred_score": 0.19313999999999998, "gold_score": 0.033333, "confidence": 0.895939, "qid": "discrepancy_severity" }, { "type": "choice", "correct": true, "kl": 0.03943001306504456, "brier": 0.009693240529999998, "confidence": 0.855536, "qid": "disposition" }, { "type": "noul", "correct": true, "kl": 0.0010183008356782434, "brier": 0.00013927804999999888, "confidence": 0.961655, "qid": "duplicate" }, { "type": "noul", "correct": true, "kl": 0.04052307842569627, "brier": 0.008378910152000006, "confidence": 0.918607, "qid": "matches_order" }, { "type": "score", "correct": true, "kl": 0.19291924909398853, "brier": 0.14568528665, "pred_score": 1.725469, "gold_score": 1.903333, "confidence": 0.390645, "qid": "urgency" } ] }, { "id": "invoice_processing_000090", "workflow": "invoice_processing", "decisions": [ { "type": "score", "correct": true, "kl": 0.033183246943588735, "brier": 0.009684726608999996, "pred_score": 2.6638230000000003, "gold_score": 2.813333, "confidence": 0.750514, "qid": "discrepancy_severity" }, { "type": "choice", "correct": true, "kl": 0.05649479682471124, "brier": 0.010560529398, "confidence": 0.709593, "qid": "disposition" }, { "type": "noul", "correct": true, "kl": 0.004780043209565352, "brier": 0.0006406768079999988, "confidence": 0.958769, "qid": "duplicate" }, { "type": "noul", "correct": true, "kl": 0.026518492566099842, "brier": 0.004080477122000001, "confidence": 0.941498, "qid": "matches_order" }, { "type": "score", "correct": true, "kl": 0.03984306735520548, "brier": 0.013277845738000003, "pred_score": 2.063871, "gold_score": 2.133333, "confidence": 0.468679, "qid": "urgency" } ] }, { "id": "invoice_processing_000091", "workflow": "invoice_processing", "decisions": [ { "type": "score", "correct": true, "kl": 0.09427986614184244, "brier": 0.013708878050999998, "pred_score": 0.36797599999999997, "gold_score": 0.106667, "confidence": 0.836336, "qid": "discrepancy_severity" }, { "type": "choice", "correct": true, "kl": 0.27339159657348483, "brier": 0.14997245751400004, "confidence": 0.636279, "qid": "disposition" }, { "type": "noul", "correct": false, "kl": 0.06025696796257232, "brier": 0.05985177216200001, "confidence": 0.556324, "qid": "duplicate" }, { "type": "noul", "correct": true, "kl": 0.020443520996187133, "brier": 0.005805893282000001, "confidence": 0.902788, "qid": "matches_order" }, { "type": "score", "correct": true, "kl": 0.016021783921059865, "brier": 0.004971548294000001, "pred_score": 1.8459690000000002, "gold_score": 1.966667, "confidence": 0.347197, "qid": "urgency" } ] }, { "id": "invoice_processing_000092", "workflow": "invoice_processing", "decisions": [ { "type": "score", "correct": false, "kl": 0.12170862346100296, "brier": 0.06319235497913077, "pred_score": 1.3396026603973397, "gold_score": 1.37, "confidence": 0.3272286727713272, "qid": "discrepancy_severity" }, { "type": "choice", "correct": true, "kl": 0.3137444076399912, "brier": 0.12799442787363313, "confidence": 0.35289864710135294, "qid": "disposition" }, { "type": "noul", "correct": true, "kl": 0.45254234991838427, "brier": 0.359058633362, "confidence": 0.529624, "qid": "duplicate" }, { "type": "noul", "correct": false, "kl": 0.15581614586651862, "brier": 0.146633868882, "confidence": 0.690771, "qid": "matches_order" }, { "type": "score", "correct": false, "kl": 0.08211560520770095, "brier": 0.04178858353300001, "pred_score": 2.3555010000000003, "gold_score": 2.1, "confidence": 0.466156, "qid": "urgency" } ] }, { "id": "invoice_processing_000093", "workflow": "invoice_processing", "decisions": [ { "type": "score", "correct": true, "kl": 0.1833628093945853, "brier": 0.04918846133499998, "pred_score": 0.636918, "gold_score": 0.99, "confidence": 0.718062, "qid": "discrepancy_severity" }, { "type": "choice", "correct": true, "kl": 0.17848292397198476, "brier": 0.06112574424999999, "confidence": 0.561866, "qid": "disposition" }, { "type": "noul", "correct": false, "kl": 0.16194345051058145, "brier": 0.141649289282, "confidence": 0.749462, "qid": "duplicate" }, { "type": "noul", "correct": true, "kl": 0.1533376121672356, "brier": 0.06208019424800002, "confidence": 0.807151, "qid": "matches_order" }, { "type": "score", "correct": true, "kl": 0.038210392648376106, "brier": 0.0031890170819999985, "pred_score": 1.101244, "gold_score": 0.983333, "confidence": 0.777084, "qid": "urgency" } ] }, { "id": "invoice_processing_000094", "workflow": "invoice_processing", "decisions": [ { "type": "score", "correct": true, "kl": 0.0690464853172622, "brier": 0.026410351255999998, "pred_score": 0.9351400000000001, "gold_score": 0.853333, "confidence": 0.492162, "qid": "discrepancy_severity" }, { "type": "choice", "correct": true, "kl": 0.017020865303501186, "brier": 0.008055769409999998, "confidence": 0.466826, "qid": "disposition" }, { "type": "noul", "correct": true, "kl": 0.0013339968684947107, "brier": 0.0002045455380000007, "confidence": 0.956554, "qid": "duplicate" }, { "type": "noul", "correct": true, "kl": 0.05735750094723931, "brier": 0.03847149649800001, "confidence": 0.73464, "qid": "matches_order" }, { "type": "score", "correct": true, "kl": 0.13369266338889982, "brier": 0.09572424512117411, "pred_score": 1.78001478001478, "gold_score": 1.87, "confidence": 0.4371824371824372, "qid": "urgency" } ] }, { "id": "invoice_processing_000095", "workflow": "invoice_processing", "decisions": [ { "type": "score", "correct": true, "kl": 0.18280419434014653, "brier": 0.08696027151800001, "pred_score": 2.572994, "gold_score": 2.916667, "confidence": 0.679268, "qid": "discrepancy_severity" }, { "type": "choice", "correct": true, "kl": 0.07778503766209928, "brier": 0.014793282294000003, "confidence": 0.731932, "qid": "disposition" }, { "type": "noul", "correct": true, "kl": 0.008169632297115625, "brier": 0.0010918464499999987, "confidence": 0.956635, "qid": "duplicate" }, { "type": "noul", "correct": true, "kl": 0.055508481609388545, "brier": 0.005831136032000005, "confidence": 0.946004, "qid": "matches_order" }, { "type": "score", "correct": true, "kl": 0.043172329617843036, "brier": 0.011386318725000004, "pred_score": 2.065287, "gold_score": 2.25, "confidence": 0.508961, "qid": "urgency" } ] }, { "id": "invoice_processing_000096", "workflow": "invoice_processing", "decisions": [ { "type": "score", "correct": true, "kl": 0.09082560448229642, "brier": 0.043315703626, "pred_score": 1.026514, "gold_score": 0.983333, "confidence": 0.427962, "qid": "discrepancy_severity" }, { "type": "choice", "correct": true, "kl": 0.05275032746915009, "brier": 0.02467588662600001, "confidence": 0.401477, "qid": "disposition" }, { "type": "noul", "correct": true, "kl": 0.002662036242731876, "brier": 0.0005090602319999971, "confidence": 0.944046, "qid": "duplicate" }, { "type": "noul", "correct": true, "kl": 0.0083910046655499, "brier": 0.006819017761999993, "confidence": 0.735058, "qid": "matches_order" }, { "type": "score", "correct": true, "kl": 0.02556359074671552, "brier": 0.016573198703893424, "pred_score": 2.1624891624891625, "gold_score": 2.05, "confidence": 0.48967548967548974, "qid": "urgency" } ] }, { "id": "invoice_processing_000097", "workflow": "invoice_processing", "decisions": [ { "type": "score", "correct": true, "kl": 0.0705787294500985, "brier": 0.009660507085, "pred_score": 0.217748, "gold_score": 0.04, "confidence": 0.884513, "qid": "discrepancy_severity" }, { "type": "choice", "correct": true, "kl": 0.03961778644925796, "brier": 0.009845625110000006, "confidence": 0.860377, "qid": "disposition" }, { "type": "noul", "correct": true, "kl": 0.006021456327548285, "brier": 0.0006561615380000015, "confidence": 0.96522, "qid": "duplicate" }, { "type": "noul", "correct": true, "kl": 0.04222230258913556, "brier": 0.009694724257999998, "confidence": 0.910377, "qid": "matches_order" }, { "type": "score", "correct": true, "kl": 0.021061509106530263, "brier": 0.0033483536430000066, "pred_score": 1.267045, "gold_score": 1.23, "confidence": 0.680252, "qid": "urgency" } ] }, { "id": "invoice_processing_000098", "workflow": "invoice_processing", "decisions": [ { "type": "score", "correct": true, "kl": 0.2869174636760359, "brier": 0.12508979465999998, "pred_score": 1.000775, "gold_score": 0.39, "confidence": 0.471154, "qid": "discrepancy_severity" }, { "type": "choice", "correct": true, "kl": 0.30093860465027283, "brier": 0.208254387963, "confidence": 0.471358, "qid": "disposition" }, { "type": "noul", "correct": true, "kl": 0.0016628725879179237, "brier": 0.00021465920000000068, "confidence": 0.962973, "qid": "duplicate" }, { "type": "noul", "correct": true, "kl": 0.11382580757071403, "brier": 0.07680221088800002, "confidence": 0.707371, "qid": "matches_order" }, { "type": "score", "correct": true, "kl": 0.009668276434854657, "brier": 0.001395741942000001, "pred_score": 1.184908, "gold_score": 1.12, "confidence": 0.711865, "qid": "urgency" } ] }, { "id": "invoice_processing_000099", "workflow": "invoice_processing", "decisions": [ { "type": "score", "correct": true, "kl": 0.18998724420786475, "brier": 0.07551271822953558, "pred_score": 1.0165549834450167, "gold_score": 0.586667, "confidence": 0.4625695374304626, "qid": "discrepancy_severity" }, { "type": "choice", "correct": true, "kl": 0.21775278356044359, "brier": 0.1542878277931217, "confidence": 0.4366055633944366, "qid": "disposition" }, { "type": "noul", "correct": true, "kl": 0.0002314209788269032, "brier": 2.486535200000019e-05, "confidence": 0.973526, "qid": "duplicate" }, { "type": "noul", "correct": true, "kl": 0.13071795249135582, "brier": 0.092260037682, "confidence": 0.685221, "qid": "matches_order" }, { "type": "score", "correct": true, "kl": 0.04796701342752625, "brier": 0.02855814228299999, "pred_score": 1.787879, "gold_score": 1.75, "confidence": 0.44657, "qid": "urgency" } ] }, { "id": "security_incidents_000000", "workflow": "security_incidents", "decisions": [ { "type": "noul", "correct": true, "kl": 0.0022394984427637696, "brier": 0.001951125511999998, "confidence": 0.668766, "qid": "credential_compromise" }, { "type": "choice", "correct": false, "kl": 0.08548476656768024, "brier": 0.07402349012299998, "confidence": 0.564218, "qid": "disposition" }, { "type": "score", "correct": true, "kl": 0.02356125106750697, "brier": 0.007044307855999999, "pred_score": 2.401039, "gold_score": 2.5, "confidence": 0.433033, "qid": "severity" }, { "type": "noul", "correct": true, "kl": 0.00028504920083953326, "brier": 0.00014992192799999902, "confidence": 0.841342, "qid": "true_positive" }, { "type": "score", "correct": true, "kl": 0.011604760557285734, "brier": 0.001530873437000002, "pred_score": 2.498491, "gold_score": 2.416667, "confidence": 0.608403, "qid": "urgency" } ] }, { "id": "security_incidents_000001", "workflow": "security_incidents", "decisions": [ { "type": "noul", "correct": true, "kl": 0.12460441701838984, "brier": 0.08701373577800003, "confidence": 0.691417, "qid": "credential_compromise" }, { "type": "choice", "correct": false, "kl": 0.37145947890989683, "brier": 0.286360858726, "confidence": 0.597589, "qid": "disposition" }, { "type": "score", "correct": true, "kl": 0.04344939302629648, "brier": 0.016666516788208983, "pred_score": 2.433877433877434, "gold_score": 2.676667, "confidence": 0.4305854305854306, "qid": "severity" }, { "type": "noul", "correct": true, "kl": 0.07548281626144168, "brier": 0.03364054849799999, "confidence": 0.820307, "qid": "true_positive" }, { "type": "score", "correct": true, "kl": 0.22960996357326802, "brier": 0.14733504546099999, "pred_score": 2.458683, "gold_score": 2.863333, "confidence": 0.5773010000000001, "qid": "urgency" } ] }, { "id": "security_incidents_000002", "workflow": "security_incidents", "decisions": [ { "type": "noul", "correct": true, "kl": 0.0037067469222530278, "brier": 0.002573600768, "confidence": 0.764128, "qid": "credential_compromise" }, { "type": "choice", "correct": false, "kl": 0.11347960667081845, "brier": 0.07410519697499998, "confidence": 0.351428, "qid": "disposition" }, { "type": "score", "correct": false, "kl": 0.12887387801956762, "brier": 0.058526114345718555, "pred_score": 2.0138149861850136, "gold_score": 2.033333, "confidence": 0.34027365972634027, "qid": "severity" }, { "type": "noul", "correct": true, "kl": 0.004258919066179902, "brier": 0.003969939618000007, "confidence": 0.644553, "qid": "true_positive" }, { "type": "score", "correct": true, "kl": 0.07707350844224109, "brier": 0.029306735444000004, "pred_score": 1.38326, "gold_score": 1.55, "confidence": 0.37522, "qid": "urgency" } ] }, { "id": "security_incidents_000003", "workflow": "security_incidents", "decisions": [ { "type": "noul", "correct": true, "kl": 0.005117852085883549, "brier": 0.004314090272000002, "confidence": 0.713111, "qid": "credential_compromise" }, { "type": "choice", "correct": true, "kl": 0.20186421076943706, "brier": 0.07892908807, "confidence": 0.556691, "qid": "disposition" }, { "type": "score", "correct": false, "kl": 0.13969662134406352, "brier": 0.06478842737499999, "pred_score": 2.470244, "gold_score": 2.05, "confidence": 0.431684, "qid": "severity" }, { "type": "noul", "correct": true, "kl": 0.060502867245841335, "brier": 0.048169734497999994, "confidence": 0.77186, "qid": "true_positive" }, { "type": "score", "correct": false, "kl": 0.14977818047544184, "brier": 0.09537475794600003, "pred_score": 2.455216, "gold_score": 2.033333, "confidence": 0.59318, "qid": "urgency" } ] }, { "id": "security_incidents_000004", "workflow": "security_incidents", "decisions": [ { "type": "noul", "correct": true, "kl": 0.1573975302366706, "brier": 0.13018977753799998, "confidence": 0.604863, "qid": "credential_compromise" }, { "type": "choice", "correct": false, "kl": 0.6847302108739786, "brier": 0.39347526610999994, "confidence": 0.54909, "qid": "disposition" }, { "type": "score", "correct": true, "kl": 0.49863084858482054, "brier": 0.129987124737, "pred_score": 1.984506, "gold_score": 1.286667, "confidence": 0.529227, "qid": "severity" }, { "type": "noul", "correct": true, "kl": 0.17920341090088177, "brier": 0.16073769607200003, "confidence": 0.543173, "qid": "true_positive" }, { "type": "score", "correct": false, "kl": 0.3952635290926223, "brier": 0.12610337870135757, "pred_score": 1.6837606837606836, "gold_score": 0.993333, "confidence": 0.38697438697438696, "qid": "urgency" } ] }, { "id": "security_incidents_000005", "workflow": "security_incidents", "decisions": [ { "type": "noul", "correct": true, "kl": 9.741978259061607e-05, "brier": 8.220184200000071e-05, "confidence": 0.699744, "qid": "credential_compromise" }, { "type": "choice", "correct": true, "kl": 0.03010934359808138, "brier": 0.017706582786000007, "confidence": 0.502656, "qid": "disposition" }, { "type": "score", "correct": false, "kl": 0.12380715361779068, "brier": 0.07396492441199999, "pred_score": 3.31337, "gold_score": 2.956667, "confidence": 0.477223, "qid": "severity" }, { "type": "noul", "correct": true, "kl": 0.005031474925723839, "brier": 0.003364296391999999, "confidence": 0.801014, "qid": "true_positive" }, { "type": "score", "correct": true, "kl": 0.10016925687313849, "brier": 0.048851332994000005, "pred_score": 2.724749, "gold_score": 2.436667, "confidence": 0.784713, "qid": "urgency" } ] }, { "id": "security_incidents_000006", "workflow": "security_incidents", "decisions": [ { "type": "noul", "correct": true, "kl": 0.007081955972852862, "brier": 0.005773610882000005, "confidence": 0.696271, "qid": "credential_compromise" }, { "type": "choice", "correct": false, "kl": 0.12904037312877978, "brier": 0.10674555763800003, "confidence": 0.592033, "qid": "disposition" }, { "type": "score", "correct": true, "kl": 0.09378681398105433, "brier": 0.03231319185444824, "pred_score": 2.7275902724097274, "gold_score": 3.08, "confidence": 0.45747654252345743, "qid": "severity" }, { "type": "noul", "correct": true, "kl": 0.00964310192246038, "brier": 0.005547204449999996, "confidence": 0.807335, "qid": "true_positive" }, { "type": "score", "correct": true, "kl": 0.011571611310445817, "brier": 0.008076615857999994, "pred_score": 2.576899, "gold_score": 2.663333, "confidence": 0.663915, "qid": "urgency" } ] }, { "id": "security_incidents_000007", "workflow": "security_incidents", "decisions": [ { "type": "noul", "correct": true, "kl": 0.05806198189476672, "brier": 0.05481757827200001, "confidence": 0.665556, "qid": "credential_compromise" }, { "type": "choice", "correct": true, "kl": 0.3147747477896714, "brier": 0.11234004395899998, "confidence": 0.527446, "qid": "disposition" }, { "type": "score", "correct": true, "kl": 0.016829119512648733, "brier": 0.003572211917000001, "pred_score": 2.408603, "gold_score": 2.28, "confidence": 0.449611, "qid": "severity" }, { "type": "noul", "correct": true, "kl": 0.03960871434190971, "brier": 0.022887301250000006, "confidence": 0.856975, "qid": "true_positive" }, { "type": "score", "correct": false, "kl": 0.12141922030844407, "brier": 0.08233591674299999, "pred_score": 2.537383, "gold_score": 2.2, "confidence": 0.634625, "qid": "urgency" } ] }, { "id": "security_incidents_000008", "workflow": "security_incidents", "decisions": [ { "type": "noul", "correct": false, "kl": 0.05445143243274664, "brier": 0.052880121631999996, "confidence": 0.629271, "qid": "credential_compromise" }, { "type": "choice", "correct": true, "kl": 0.15256239630670557, "brier": 0.06365146309000003, "confidence": 0.632565, "qid": "disposition" }, { "type": "score", "correct": false, "kl": 0.10909817061799454, "brier": 0.061240211873999986, "pred_score": 2.6429, "gold_score": 2.286667, "confidence": 0.494422, "qid": "severity" }, { "type": "noul", "correct": true, "kl": 0.05017178494238314, "brier": 0.04075798005000002, "confidence": 0.759422, "qid": "true_positive" }, { "type": "score", "correct": true, "kl": 0.08006995683382713, "brier": 0.029111176963999997, "pred_score": 2.405628, "gold_score": 2.096667, "confidence": 0.551844, "qid": "urgency" } ] }, { "id": "security_incidents_000009", "workflow": "security_incidents", "decisions": [ { "type": "noul", "correct": true, "kl": 0.009141526753244887, "brier": 0.008541767807999999, "confidence": 0.648685, "qid": "credential_compromise" }, { "type": "choice", "correct": false, "kl": 0.1867935487153614, "brier": 0.106278786618, "confidence": 0.356128, "qid": "disposition" }, { "type": "score", "correct": true, "kl": 0.043014203696556785, "brier": 0.010682717323999988, "pred_score": 1.8645450000000001, "gold_score": 1.766667, "confidence": 0.45068, "qid": "severity" }, { "type": "noul", "correct": true, "kl": 0.00019242192501926943, "brier": 0.00018305497800000077, "confidence": 0.6071, "qid": "true_positive" }, { "type": "score", "correct": true, "kl": 0.10122903202603202, "brier": 0.038718451683, "pred_score": 1.408149, "gold_score": 1.45, "confidence": 0.37956, "qid": "urgency" } ] }, { "id": "security_incidents_000010", "workflow": "security_incidents", "decisions": [ { "type": "noul", "correct": false, "kl": 0.04022079713177172, "brier": 0.039957383432000004, "confidence": 0.524679, "qid": "credential_compromise" }, { "type": "choice", "correct": true, "kl": 0.0832766917019424, "brier": 0.018139423550000006, "confidence": 0.586306, "qid": "disposition" }, { "type": "score", "correct": true, "kl": 0.01808175152850386, "brier": 0.0033706057959999995, "pred_score": 2.649109, "gold_score": 2.53, "confidence": 0.5357850000000001, "qid": "severity" }, { "type": "noul", "correct": false, "kl": 0.013443984593442301, "brier": 0.013422755858000008, "confidence": 0.531923, "qid": "true_positive" }, { "type": "score", "correct": true, "kl": 0.008722887517153784, "brier": 0.0038996809820000026, "pred_score": 2.101013, "gold_score": 2.183333, "confidence": 0.393551, "qid": "urgency" } ] }, { "id": "security_incidents_000011", "workflow": "security_incidents", "decisions": [ { "type": "noul", "correct": false, "kl": 0.06584523069740644, "brier": 0.06525367128199996, "confidence": 0.580629, "qid": "credential_compromise" }, { "type": "choice", "correct": true, "kl": 0.08432120066580698, "brier": 0.03121675052199999, "confidence": 0.545022, "qid": "disposition" }, { "type": "score", "correct": false, "kl": 0.12989557508145705, "brier": 0.07963574642200001, "pred_score": 3.1386890000000003, "gold_score": 2.756667, "confidence": 0.450043, "qid": "severity" }, { "type": "noul", "correct": false, "kl": 0.04945934590742512, "brier": 0.04820326201799998, "confidence": 0.621914, "qid": "true_positive" }, { "type": "score", "correct": true, "kl": 0.053739891595561945, "brier": 0.038343731533, "pred_score": 2.383855, "gold_score": 2.183333, "confidence": 0.572009, "qid": "urgency" } ] }, { "id": "security_incidents_000012", "workflow": "security_incidents", "decisions": [ { "type": "noul", "correct": true, "kl": 4.351882041736836e-06, "brier": 4.310047999999983e-06, "confidence": 0.548532, "qid": "credential_compromise" }, { "type": "choice", "correct": true, "kl": 0.007547001753918085, "brier": 0.005597918621999994, "confidence": 0.473144, "qid": "disposition" }, { "type": "score", "correct": true, "kl": 0.06012815101272749, "brier": 0.019488697511999997, "pred_score": 1.215307, "gold_score": 1.3, "confidence": 0.468022, "qid": "severity" }, { "type": "noul", "correct": true, "kl": 0.0006578333280581362, "brier": 0.0006565963219999998, "confidence": 0.515214, "qid": "true_positive" }, { "type": "score", "correct": true, "kl": 0.029147454078341267, "brier": 0.009754971896108319, "pred_score": 1.4231704231704232, "gold_score": 1.416667, "confidence": 0.46960446960446967, "qid": "urgency" } ] }, { "id": "security_incidents_000013", "workflow": "security_incidents", "decisions": [ { "type": "noul", "correct": true, "kl": 0.005128008708218522, "brier": 0.0037963412479999946, "confidence": 0.739765, "qid": "credential_compromise" }, { "type": "choice", "correct": false, "kl": 0.06179084384363548, "brier": 0.03829290923361615, "confidence": 0.3977046022953977, "qid": "disposition" }, { "type": "score", "correct": true, "kl": 0.026442055742212457, "brier": 0.004050504262764046, "pred_score": 1.6987266987266987, "gold_score": 1.583333, "confidence": 0.49188749188749187, "qid": "severity" }, { "type": "noul", "correct": true, "kl": 0.019453662570154434, "brier": 0.01710103192200001, "confidence": 0.640864, "qid": "true_positive" }, { "type": "score", "correct": true, "kl": 0.07010066593753744, "brier": 0.044287391832878976, "pred_score": 1.3940886059113944, "gold_score": 1.216667, "confidence": 0.3893466106533894, "qid": "urgency" } ] }, { "id": "security_incidents_000014", "workflow": "security_incidents", "decisions": [ { "type": "noul", "correct": true, "kl": 0.00010067615039678363, "brier": 6.534531200000062e-05, "confidence": 0.794284, "qid": "credential_compromise" }, { "type": "choice", "correct": true, "kl": 0.020356948634494897, "brier": 0.016451743608000006, "confidence": 0.549092, "qid": "disposition" }, { "type": "score", "correct": true, "kl": 0.07940392719045787, "brier": 0.041163030244999996, "pred_score": 3.112276, "gold_score": 2.913333, "confidence": 0.55713, "qid": "severity" }, { "type": "noul", "correct": true, "kl": 0.001306534680145452, "brier": 0.0006358178, "confidence": 0.85217, "qid": "true_positive" }, { "type": "score", "correct": true, "kl": 0.02270211020134258, "brier": 0.009706144659000002, "pred_score": 2.753819, "gold_score": 2.71, "confidence": 0.811123, "qid": "urgency" } ] }, { "id": "security_incidents_000015", "workflow": "security_incidents", "decisions": [ { "type": "noul", "correct": true, "kl": 0.0010237843679949017, "brier": 0.0009985980500000027, "confidence": 0.585678, "qid": "credential_compromise" }, { "type": "choice", "correct": true, "kl": 0.017073507703340325, "brier": 0.0038995287589999985, "confidence": 0.659658, "qid": "disposition" }, { "type": "score", "correct": false, "kl": 0.0710977523450398, "brier": 0.032911074177999995, "pred_score": 1.853541, "gold_score": 1.543333, "confidence": 0.399915, "qid": "severity" }, { "type": "noul", "correct": true, "kl": 0.001180647792172737, "brier": 0.000956156450000002, "confidence": 0.725198, "qid": "true_positive" }, { "type": "score", "correct": true, "kl": 0.07352163595312083, "brier": 0.042190742691000004, "pred_score": 2.040232, "gold_score": 1.84, "confidence": 0.432772, "qid": "urgency" } ] }, { "id": "security_incidents_000020", "workflow": "security_incidents", "decisions": [ { "type": "noul", "correct": true, "kl": 0.006054375707237533, "brier": 0.0060174838079999965, "confidence": 0.518481, "qid": "credential_compromise" }, { "type": "choice", "correct": true, "kl": 0.04957486489486714, "brier": 0.016372195436, "confidence": 0.616701, "qid": "disposition" }, { "type": "score", "correct": true, "kl": 0.021025141065805344, "brier": 0.007274679278999994, "pred_score": 2.242028, "gold_score": 2.1, "confidence": 0.492834, "qid": "severity" }, { "type": "noul", "correct": true, "kl": 0.01801084751459346, "brier": 0.016792846847999997, "confidence": 0.658299, "qid": "true_positive" }, { "type": "score", "correct": true, "kl": 0.14115867188868217, "brier": 0.075545886854, "pred_score": 2.093248, "gold_score": 1.716667, "confidence": 0.403706, "qid": "urgency" } ] }, { "id": "security_incidents_000021", "workflow": "security_incidents", "decisions": [ { "type": "noul", "correct": true, "kl": 0.0010075197534272495, "brier": 0.0009544954320000028, "confidence": 0.621846, "qid": "credential_compromise" }, { "type": "choice", "correct": true, "kl": 0.03667764684246596, "brier": 0.012632093862, "confidence": 0.601572, "qid": "disposition" }, { "type": "score", "correct": true, "kl": 0.03712495350371327, "brier": 0.007794897738999999, "pred_score": 2.845436, "gold_score": 2.7, "confidence": 0.635265, "qid": "severity" }, { "type": "noul", "correct": true, "kl": 0.0014325681717874926, "brier": 0.0012728030579999999, "confidence": 0.675227, "qid": "true_positive" }, { "type": "score", "correct": true, "kl": 0.054870153190827414, "brier": 0.036520668327, "pred_score": 2.399282, "gold_score": 2.216667, "confidence": 0.565882, "qid": "urgency" } ] }, { "id": "security_incidents_000022", "workflow": "security_incidents", "decisions": [ { "type": "noul", "correct": true, "kl": 1.9896860715659702e-06, "brier": 1.909058000000022e-06, "confidence": 0.600977, "qid": "credential_compromise" }, { "type": "choice", "correct": true, "kl": 0.01665189666332293, "brier": 0.005233465249999997, "confidence": 0.553107, "qid": "disposition" }, { "type": "score", "correct": false, "kl": 0.08220600543183083, "brier": 0.03674683478500001, "pred_score": 2.60237, "gold_score": 2.283333, "confidence": 0.491739, "qid": "severity" }, { "type": "noul", "correct": true, "kl": 2.006413990833294e-06, "brier": 2.004002000000119e-06, "confidence": 0.517668, "qid": "true_positive" }, { "type": "score", "correct": true, "kl": 0.09065506141173141, "brier": 0.05407323640400001, "pred_score": 2.119296, "gold_score": 1.816667, "confidence": 0.396939, "qid": "urgency" } ] }, { "id": "security_incidents_000023", "workflow": "security_incidents", "decisions": [ { "type": "noul", "correct": true, "kl": 0.01607610697052554, "brier": 0.010313639441999995, "confidence": 0.821811, "qid": "credential_compromise" }, { "type": "choice", "correct": false, "kl": 0.2124666466870224, "brier": 0.12824823724994944, "confidence": 0.3719283719283719, "qid": "disposition" }, { "type": "score", "correct": true, "kl": 0.02992471253766088, "brier": 0.008575110381256926, "pred_score": 2.5645285645285645, "gold_score": 2.683333, "confidence": 0.491994491994492, "qid": "severity" }, { "type": "noul", "correct": true, "kl": 0.01558456312669848, "brier": 0.013196625799999999, "confidence": 0.72123, "qid": "true_positive" }, { "type": "score", "correct": false, "kl": 0.08543529976410677, "brier": 0.036766098145786456, "pred_score": 1.6669766669766668, "gold_score": 2.0, "confidence": 0.3497893497893498, "qid": "urgency" } ] }, { "id": "security_incidents_000024", "workflow": "security_incidents", "decisions": [ { "type": "noul", "correct": true, "kl": 0.00019693640929662783, "brier": 0.00019685248199999942, "confidence": 0.506746, "qid": "credential_compromise" }, { "type": "choice", "correct": true, "kl": 0.03742791809449756, "brier": 0.015787824314000005, "confidence": 0.568823, "qid": "disposition" }, { "type": "score", "correct": true, "kl": 0.016926242160136527, "brier": 0.0007489114160000006, "pred_score": 2.605367, "gold_score": 2.666667, "confidence": 0.525858, "qid": "severity" }, { "type": "noul", "correct": true, "kl": 0.0005956504192642827, "brier": 0.0005691263219999977, "confidence": 0.599798, "qid": "true_positive" }, { "type": "score", "correct": false, "kl": 0.057679609761615055, "brier": 0.013113992477999997, "pred_score": 2.168115, "gold_score": 2.216667, "confidence": 0.412082, "qid": "urgency" } ] }, { "id": "security_incidents_000025", "workflow": "security_incidents", "decisions": [ { "type": "noul", "correct": true, "kl": 0.019188921864647546, "brier": 0.018825372721999998, "confidence": 0.597019, "qid": "credential_compromise" }, { "type": "choice", "correct": true, "kl": 0.06582807077325895, "brier": 0.022765720145999997, "confidence": 0.6049180000000001, "qid": "disposition" }, { "type": "score", "correct": false, "kl": 0.05013416871495098, "brier": 0.008417878226000004, "pred_score": 2.450347, "gold_score": 2.2, "confidence": 0.40797, "qid": "severity" }, { "type": "noul", "correct": true, "kl": 0.019773336041300146, "brier": 0.017195480352, "confidence": 0.709391, "qid": "true_positive" }, { "type": "score", "correct": true, "kl": 0.016956731477454245, "brier": 0.005535193286967061, "pred_score": 2.306249306249306, "gold_score": 2.183333, "confidence": 0.496016496016496, "qid": "urgency" } ] }, { "id": "security_incidents_000026", "workflow": "security_incidents", "decisions": [ { "type": "noul", "correct": true, "kl": 0.003964068434859497, "brier": 0.003916302002000003, "confidence": 0.539082, "qid": "credential_compromise" }, { "type": "choice", "correct": true, "kl": 0.03403184223421547, "brier": 0.018467120071000008, "confidence": 0.612058, "qid": "disposition" }, { "type": "score", "correct": false, "kl": 0.17216684962913884, "brier": 0.082053496167, "pred_score": 2.787112, "gold_score": 2.316667, "confidence": 0.525975, "qid": "severity" }, { "type": "noul", "correct": true, "kl": 0.015355030213044657, "brier": 0.01512164841799999, "confidence": 0.586953, "qid": "true_positive" }, { "type": "score", "correct": false, "kl": 0.05295901856671975, "brier": 0.036318177298702674, "pred_score": 2.232839767160233, "gold_score": 2.0, "confidence": 0.4404645595354405, "qid": "urgency" } ] }, { "id": "security_incidents_000027", "workflow": "security_incidents", "decisions": [ { "type": "noul", "correct": true, "kl": 0.039643146441674895, "brier": 0.038112291871999986, "confidence": 0.638044, "qid": "credential_compromise" }, { "type": "choice", "correct": true, "kl": 0.09383553497978561, "brier": 0.03354770900299999, "confidence": 0.619096, "qid": "disposition" }, { "type": "score", "correct": true, "kl": 0.29001582150035743, "brier": 0.09345070537, "pred_score": 2.245544, "gold_score": 1.65, "confidence": 0.485736, "qid": "severity" }, { "type": "noul", "correct": true, "kl": 0.021563625278747778, "brier": 0.016617738817999984, "confidence": 0.76782, "qid": "true_positive" }, { "type": "score", "correct": false, "kl": 0.1589133152159909, "brier": 0.10895337834199999, "pred_score": 2.329074, "gold_score": 1.933333, "confidence": 0.503788, "qid": "urgency" } ] }, { "id": "security_incidents_000028", "workflow": "security_incidents", "decisions": [ { "type": "noul", "correct": true, "kl": 0.0008477630103476859, "brier": 0.0006273590420000003, "confidence": 0.748956, "qid": "credential_compromise" }, { "type": "choice", "correct": true, "kl": 0.030548441806334904, "brier": 0.008901381991999996, "confidence": 0.64059, "qid": "disposition" }, { "type": "score", "correct": true, "kl": 0.03701905461988672, "brier": 0.021121283791999988, "pred_score": 3.7169869999999996, "gold_score": 3.563333, "confidence": 0.782534, "qid": "severity" }, { "type": "noul", "correct": true, "kl": 0.004614864549765094, "brier": 0.002199967112000001, "confidence": 0.850167, "qid": "true_positive" }, { "type": "score", "correct": true, "kl": 0.017541448232981455, "brier": 0.008114208214000008, "pred_score": 2.802129, "gold_score": 2.726667, "confidence": 0.850495, "qid": "urgency" } ] }, { "id": "security_incidents_000029", "workflow": "security_incidents", "decisions": [ { "type": "noul", "correct": true, "kl": 0.013157588520478672, "brier": 0.01136579645000001, "confidence": 0.657948, "qid": "credential_compromise" }, { "type": "choice", "correct": true, "kl": 0.1435667131549403, "brier": 0.07288874208799999, "confidence": 0.564905, "qid": "disposition" }, { "type": "score", "correct": true, "kl": 0.15161249404706412, "brier": 0.044613428851, "pred_score": 3.103337, "gold_score": 2.733333, "confidence": 0.616371, "qid": "severity" }, { "type": "noul", "correct": true, "kl": 0.030908639682918043, "brier": 0.02570007232800002, "confidence": 0.740025, "qid": "true_positive" }, { "type": "score", "correct": true, "kl": 0.19617370510618468, "brier": 0.12757754634999996, "pred_score": 2.615572, "gold_score": 2.166667, "confidence": 0.712942, "qid": "urgency" } ] }, { "id": "security_incidents_000030", "workflow": "security_incidents", "decisions": [ { "type": "noul", "correct": true, "kl": 0.08594606714195338, "brier": 0.06858993144200001, "confidence": 0.778522, "qid": "credential_compromise" }, { "type": "choice", "correct": true, "kl": 0.2901580615205124, "brier": 0.07014222912899998, "confidence": 0.603798, "qid": "disposition" }, { "type": "score", "correct": true, "kl": 0.13964164681254607, "brier": 0.04940777703700001, "pred_score": 3.3833849999999996, "gold_score": 3.2, "confidence": 0.500284, "qid": "severity" }, { "type": "noul", "correct": true, "kl": 0.058065617283638765, "brier": 0.03185388960800001, "confidence": 0.872869, "qid": "true_positive" }, { "type": "score", "correct": true, "kl": 0.20205408148072293, "brier": 0.10036547296400002, "pred_score": 2.831718, "gold_score": 2.533333, "confidence": 0.862908, "qid": "urgency" } ] }, { "id": "security_incidents_000031", "workflow": "security_incidents", "decisions": [ { "type": "noul", "correct": true, "kl": 0.0908815403980385, "brier": 0.055421858312, "confidence": 0.746867, "qid": "credential_compromise" }, { "type": "choice", "correct": true, "kl": 0.08205913790666905, "brier": 0.06105739811325558, "confidence": 0.47127552872447126, "qid": "disposition" }, { "type": "score", "correct": true, "kl": 0.04613730996145479, "brier": 0.018572934470000014, "pred_score": 2.315477, "gold_score": 2.34, "confidence": 0.446428, "qid": "severity" }, { "type": "noul", "correct": true, "kl": 0.04316319533521348, "brier": 0.012981438450000018, "confidence": 0.886102, "qid": "true_positive" }, { "type": "score", "correct": true, "kl": 0.07588181168760567, "brier": 0.04670769102099998, "pred_score": 2.5673600000000003, "gold_score": 2.796667, "confidence": 0.661437, "qid": "urgency" } ] }, { "id": "security_incidents_000032", "workflow": "security_incidents", "decisions": [ { "type": "noul", "correct": true, "kl": 0.10268639921967297, "brier": 0.05483810913799998, "confidence": 0.774413, "qid": "credential_compromise" }, { "type": "choice", "correct": false, "kl": 0.5915696518812255, "brier": 0.37935045595899997, "confidence": 0.358729, "qid": "disposition" }, { "type": "score", "correct": true, "kl": 0.26361055082889034, "brier": 0.097772166399, "pred_score": 1.583782, "gold_score": 1.19, "confidence": 0.431228, "qid": "severity" }, { "type": "noul", "correct": true, "kl": 0.1515956474900914, "brier": 0.10634995281800004, "confidence": 0.679403, "qid": "true_positive" }, { "type": "score", "correct": false, "kl": 0.3998330776360788, "brier": 0.16759858633, "pred_score": 1.307282, "gold_score": 0.636667, "confidence": 0.477944, "qid": "urgency" } ] }, { "id": "security_incidents_000033", "workflow": "security_incidents", "decisions": [ { "type": "noul", "correct": true, "kl": 0.013596059307517158, "brier": 0.013070534562000007, "confidence": 0.569159, "qid": "credential_compromise" }, { "type": "choice", "correct": true, "kl": 0.005447954236618084, "brier": 0.0019656189580000006, "confidence": 0.581335, "qid": "disposition" }, { "type": "score", "correct": true, "kl": 0.02683206147434583, "brier": 0.0038071849101974655, "pred_score": 1.6051436051436052, "gold_score": 1.5, "confidence": 0.4080844080844081, "qid": "severity" }, { "type": "noul", "correct": true, "kl": 0.0013644750455065122, "brier": 0.001358455688, "confidence": 0.523938, "qid": "true_positive" }, { "type": "score", "correct": false, "kl": 0.04558269050395897, "brier": 0.02175123702832922, "pred_score": 1.7623617623617625, "gold_score": 1.516667, "confidence": 0.441969441969442, "qid": "urgency" } ] }, { "id": "security_incidents_000034", "workflow": "security_incidents", "decisions": [ { "type": "noul", "correct": true, "kl": 0.03831195493753309, "brier": 0.03446677635199999, "confidence": 0.697943, "qid": "credential_compromise" }, { "type": "choice", "correct": false, "kl": 0.06682058743981, "brier": 0.05638082020899999, "confidence": 0.471197, "qid": "disposition" }, { "type": "score", "correct": true, "kl": 0.08041881606206105, "brier": 0.05442215849212209, "pred_score": 3.284311284311284, "gold_score": 3.05, "confidence": 0.47514847514847514, "qid": "severity" }, { "type": "noul", "correct": true, "kl": 0.01768385970330101, "brier": 0.011883886112000005, "confidence": 0.810417, "qid": "true_positive" }, { "type": "score", "correct": true, "kl": 0.04296802062790596, "brier": 0.024233566025999995, "pred_score": 2.7129790000000003, "gold_score": 2.533333, "confidence": 0.776798, "qid": "urgency" } ] }, { "id": "security_incidents_000035", "workflow": "security_incidents", "decisions": [ { "type": "noul", "correct": true, "kl": 0.0653070050870253, "brier": 0.056642375042000016, "confidence": 0.731622, "qid": "credential_compromise" }, { "type": "choice", "correct": false, "kl": 0.26736177573613007, "brier": 0.09362593103400002, "confidence": 0.465704, "qid": "disposition" }, { "type": "score", "correct": true, "kl": 0.20627285346221072, "brier": 0.054988538186000004, "pred_score": 3.539126, "gold_score": 3.13, "confidence": 0.607175, "qid": "severity" }, { "type": "noul", "correct": true, "kl": 0.08638945165600873, "brier": 0.058914400322, "confidence": 0.831631, "qid": "true_positive" }, { "type": "score", "correct": true, "kl": 0.2331548398997578, "brier": 0.11142279854300001, "pred_score": 2.809707, "gold_score": 2.45, "confidence": 0.851236, "qid": "urgency" } ] }, { "id": "security_incidents_000036", "workflow": "security_incidents", "decisions": [ { "type": "noul", "correct": true, "kl": 0.003230050654701694, "brier": 0.0030739072320000024, "confidence": 0.622537, "qid": "credential_compromise" }, { "type": "choice", "correct": true, "kl": 0.047219666071719256, "brier": 0.015500434645999999, "confidence": 0.598351, "qid": "disposition" }, { "type": "score", "correct": true, "kl": 0.04179434960912394, "brier": 0.02376017043300001, "pred_score": 2.687811, "gold_score": 2.47, "confidence": 0.528645, "qid": "severity" }, { "type": "noul", "correct": true, "kl": 0.015194133749153724, "brier": 0.013698435200000008, "confidence": 0.68276, "qid": "true_positive" }, { "type": "score", "correct": false, "kl": 0.07902673055108761, "brier": 0.05700898785900003, "pred_score": 2.3647810000000002, "gold_score": 2.083333, "confidence": 0.528889, "qid": "urgency" } ] }, { "id": "security_incidents_000037", "workflow": "security_incidents", "decisions": [ { "type": "noul", "correct": true, "kl": 0.004689188372506516, "brier": 0.004642046658000005, "confidence": 0.564844, "qid": "credential_compromise" }, { "type": "choice", "correct": true, "kl": 0.014705852345118172, "brier": 0.0029964683460000003, "confidence": 0.644919, "qid": "disposition" }, { "type": "score", "correct": true, "kl": 0.00900106070336346, "brier": 0.0019207088936334714, "pred_score": 1.7920732079267923, "gold_score": 1.723333, "confidence": 0.42641357358642645, "qid": "severity" }, { "type": "noul", "correct": true, "kl": 0.015366235320674382, "brier": 0.013170644999999991, "confidence": 0.714483, "qid": "true_positive" }, { "type": "score", "correct": false, "kl": 0.15819692680183473, "brier": 0.08076153008243409, "pred_score": 1.968068031931968, "gold_score": 1.55, "confidence": 0.4465085534914465, "qid": "urgency" } ] }, { "id": "security_incidents_000038", "workflow": "security_incidents", "decisions": [ { "type": "noul", "correct": false, "kl": 0.032123101966674644, "brier": 0.03153665436800001, "confidence": 0.602239, "qid": "credential_compromise" }, { "type": "choice", "correct": true, "kl": 0.008092387340985895, "brier": 0.0013178623179999989, "confidence": 0.600932, "qid": "disposition" }, { "type": "score", "correct": true, "kl": 0.0339944169734761, "brier": 0.011728829692999989, "pred_score": 2.775017, "gold_score": 2.9, "confidence": 0.524205, "qid": "severity" }, { "type": "noul", "correct": true, "kl": 0.008066627932639028, "brier": 0.005921113841999997, "confidence": 0.738922, "qid": "true_positive" }, { "type": "score", "correct": true, "kl": 0.02113987911409497, "brier": 0.004836706698000004, "pred_score": 2.492528, "gold_score": 2.566667, "confidence": 0.598753, "qid": "urgency" } ] }, { "id": "security_incidents_000039", "workflow": "security_incidents", "decisions": [ { "type": "noul", "correct": true, "kl": 0.0046423135776592, "brier": 0.0033757901119999996, "confidence": 0.774417, "qid": "credential_compromise" }, { "type": "choice", "correct": false, "kl": 0.09845361399163716, "brier": 0.05498945896800001, "confidence": 0.509795, "qid": "disposition" }, { "type": "score", "correct": true, "kl": 0.08404700436783195, "brier": 0.032513034446, "pred_score": 2.865028, "gold_score": 2.663333, "confidence": 0.492686, "qid": "severity" }, { "type": "noul", "correct": true, "kl": 0.0007098251819334395, "brier": 0.0003442688000000004, "confidence": 0.86312, "qid": "true_positive" }, { "type": "score", "correct": true, "kl": 0.07410636930873155, "brier": 0.042453013725000016, "pred_score": 2.703331, "gold_score": 2.46, "confidence": 0.76846, "qid": "urgency" } ] }, { "id": "security_incidents_000040", "workflow": "security_incidents", "decisions": [ { "type": "noul", "correct": false, "kl": 0.0009418968349582703, "brier": 0.0009417799999999973, "confidence": 0.505033, "qid": "credential_compromise" }, { "type": "choice", "correct": true, "kl": 0.08139407970964932, "brier": 0.028500354542, "confidence": 0.623329, "qid": "disposition" }, { "type": "score", "correct": true, "kl": 0.006814820084272238, "brier": 0.0043151562830000025, "pred_score": 1.714724, "gold_score": 1.726667, "confidence": 0.372712, "qid": "severity" }, { "type": "noul", "correct": true, "kl": 0.027967374951703622, "brier": 0.019871806082000013, "confidence": 0.799679, "qid": "true_positive" }, { "type": "score", "correct": true, "kl": 0.1404743367872073, "brier": 0.06644532147999999, "pred_score": 2.04887, "gold_score": 1.616667, "confidence": 0.402451, "qid": "urgency" } ] }, { "id": "security_incidents_000041", "workflow": "security_incidents", "decisions": [ { "type": "noul", "correct": true, "kl": 0.028939405367619186, "brier": 0.020584205000000005, "confidence": 0.731883, "qid": "credential_compromise" }, { "type": "choice", "correct": false, "kl": 0.15022827316412585, "brier": 0.08113041845000003, "confidence": 0.529695, "qid": "disposition" }, { "type": "score", "correct": false, "kl": 0.33574182674133624, "brier": 0.16884706735026125, "pred_score": 1.298133298133298, "gold_score": 0.636667, "confidence": 0.4387934387934388, "qid": "severity" }, { "type": "noul", "correct": false, "kl": 0.10453248666811193, "brier": 0.103202422562, "confidence": 0.560492, "qid": "true_positive" }, { "type": "score", "correct": true, "kl": 0.24915853008981936, "brier": 0.086466098059, "pred_score": 1.4479980000000001, "gold_score": 0.876667, "confidence": 0.419566, "qid": "urgency" } ] }, { "id": "security_incidents_000042", "workflow": "security_incidents", "decisions": [ { "type": "noul", "correct": false, "kl": 0.020474437652662048, "brier": 0.020403212018000015, "confidence": 0.551003, "qid": "credential_compromise" }, { "type": "choice", "correct": false, "kl": 0.1846985511903863, "brier": 0.083934582934, "confidence": 0.528399, "qid": "disposition" }, { "type": "score", "correct": true, "kl": 0.07438753684245716, "brier": 0.04902332113399999, "pred_score": 2.773684, "gold_score": 2.543333, "confidence": 0.597586, "qid": "severity" }, { "type": "noul", "correct": true, "kl": 0.0012919877956404022, "brier": 0.0012903199999999978, "confidence": 0.5254, "qid": "true_positive" }, { "type": "score", "correct": false, "kl": 0.02560273080118179, "brier": 0.009761105318, "pred_score": 2.1219520000000003, "gold_score": 2.163333, "confidence": 0.421211, "qid": "urgency" } ] }, { "id": "security_incidents_000043", "workflow": "security_incidents", "decisions": [ { "type": "noul", "correct": true, "kl": 0.015339680189412182, "brier": 0.008458322047999997, "confidence": 0.811635, "qid": "credential_compromise" }, { "type": "choice", "correct": true, "kl": 0.03151895630414071, "brier": 0.012132628253999989, "confidence": 0.651738, "qid": "disposition" }, { "type": "score", "correct": true, "kl": 0.052469385852131525, "brier": 0.032599520904, "pred_score": 3.493129, "gold_score": 3.683333, "confidence": 0.591052, "qid": "severity" }, { "type": "noul", "correct": true, "kl": 8.643282806661571e-07, "brier": 2.9184799999999e-07, "confidence": 0.907049, "qid": "true_positive" }, { "type": "score", "correct": true, "kl": 0.01225459965890073, "brier": 0.0033067574100000005, "pred_score": 2.850736, "gold_score": 2.806667, "confidence": 0.875876, "qid": "urgency" } ] }, { "id": "security_incidents_000044", "workflow": "security_incidents", "decisions": [ { "type": "noul", "correct": false, "kl": 0.03513111834673441, "brier": 0.034287424712, "confidence": 0.614267, "qid": "credential_compromise" }, { "type": "choice", "correct": true, "kl": 0.025236740901295532, "brier": 0.015915018450000006, "confidence": 0.552831, "qid": "disposition" }, { "type": "score", "correct": false, "kl": 0.5022818301836116, "brier": 0.26226737797272826, "pred_score": 1.8642408642408643, "gold_score": 2.76, "confidence": 0.3385083385083385, "qid": "severity" }, { "type": "noul", "correct": false, "kl": 0.01717070285761438, "brier": 0.017136188191999992, "confidence": 0.525897, "qid": "true_positive" }, { "type": "score", "correct": true, "kl": 0.2423866896597338, "brier": 0.12037060201399999, "pred_score": 1.694203, "gold_score": 2.233333, "confidence": 0.399812, "qid": "urgency" } ] }, { "id": "security_incidents_000045", "workflow": "security_incidents", "decisions": [ { "type": "noul", "correct": true, "kl": 0.0027604555880099066, "brier": 0.0026965699219999965, "confidence": 0.563281, "qid": "credential_compromise" }, { "type": "choice", "correct": true, "kl": 0.010852461679690097, "brier": 0.002885074542000002, "confidence": 0.624143, "qid": "disposition" }, { "type": "score", "correct": true, "kl": 0.03938141928682358, "brier": 0.019266031455999998, "pred_score": 1.7623060000000002, "gold_score": 1.883333, "confidence": 0.398293, "qid": "severity" }, { "type": "noul", "correct": true, "kl": 4.9357330601166975e-06, "brier": 4.380800000000054e-06, "confidence": 0.668147, "qid": "true_positive" }, { "type": "score", "correct": true, "kl": 0.015918947006888916, "brier": 0.011078632574000002, "pred_score": 1.9330060000000002, "gold_score": 1.916667, "confidence": 0.415613, "qid": "urgency" } ] }, { "id": "security_incidents_000046", "workflow": "security_incidents", "decisions": [ { "type": "noul", "correct": true, "kl": 0.0001465666487479502, "brier": 0.00013431605000000115, "confidence": 0.641805, "qid": "credential_compromise" }, { "type": "choice", "correct": true, "kl": 0.04339954702623302, "brier": 0.03509489472599999, "confidence": 0.478804, "qid": "disposition" }, { "type": "score", "correct": true, "kl": 0.05398253299950842, "brier": 0.018672471014999997, "pred_score": 2.973202, "gold_score": 2.736667, "confidence": 0.548922, "qid": "severity" }, { "type": "noul", "correct": true, "kl": 0.0037641836039178436, "brier": 0.0028075521779999998, "confidence": 0.7392, "qid": "true_positive" }, { "type": "score", "correct": true, "kl": 0.033055346289622714, "brier": 0.016190226106999998, "pred_score": 2.596128, "gold_score": 2.416667, "confidence": 0.691105, "qid": "urgency" } ] }, { "id": "security_incidents_000047", "workflow": "security_incidents", "decisions": [ { "type": "noul", "correct": true, "kl": 0.0038790989997775083, "brier": 0.003647946527999999, "confidence": 0.607292, "qid": "credential_compromise" }, { "type": "choice", "correct": true, "kl": 0.06367486457872257, "brier": 0.030789144384999995, "confidence": 0.475239, "qid": "disposition" }, { "type": "score", "correct": true, "kl": 0.02125338656230851, "brier": 0.010138954186258025, "pred_score": 2.0602459397540605, "gold_score": 1.966667, "confidence": 0.4315705684294316, "qid": "severity" }, { "type": "noul", "correct": true, "kl": 2.8964060219275993e-05, "brier": 2.8940832000000027e-05, "confidence": 0.512863, "qid": "true_positive" }, { "type": "score", "correct": true, "kl": 0.044831498461268056, "brier": 0.020673024333205208, "pred_score": 1.6634356634356633, "gold_score": 1.866667, "confidence": 0.3715033715033715, "qid": "urgency" } ] }, { "id": "security_incidents_000048", "workflow": "security_incidents", "decisions": [ { "type": "noul", "correct": false, "kl": 0.16827476756012805, "brier": 0.15257177520200002, "confidence": 0.726199, "qid": "credential_compromise" }, { "type": "choice", "correct": true, "kl": 0.07241259689662263, "brier": 0.02373054090800001, "confidence": 0.553627, "qid": "disposition" }, { "type": "score", "correct": true, "kl": 0.031393571149861535, "brier": 0.012146684688000002, "pred_score": 2.88145, "gold_score": 2.973333, "confidence": 0.660983, "qid": "severity" }, { "type": "noul", "correct": false, "kl": 0.10349981694314192, "brier": 0.101261700338, "confidence": 0.625013, "qid": "true_positive" }, { "type": "score", "correct": false, "kl": 0.17368360288797646, "brier": 0.09703518905516825, "pred_score": 1.9770740229259773, "gold_score": 2.45, "confidence": 0.41230458769541234, "qid": "urgency" } ] }, { "id": "security_incidents_000049", "workflow": "security_incidents", "decisions": [ { "type": "noul", "correct": true, "kl": 0.0006603209712461831, "brier": 0.0006363171379999994, "confidence": 0.60117, "qid": "credential_compromise" }, { "type": "choice", "correct": true, "kl": 0.026519005639105052, "brier": 0.009222296713999994, "confidence": 0.549171, "qid": "disposition" }, { "type": "score", "correct": false, "kl": 0.10857119489000597, "brier": 0.052635228042434236, "pred_score": 1.7235162764837235, "gold_score": 1.353333, "confidence": 0.49114250885749117, "qid": "severity" }, { "type": "noul", "correct": true, "kl": 8.372911533224663e-05, "brier": 8.274124799999982e-05, "confidence": 0.556432, "qid": "true_positive" }, { "type": "score", "correct": true, "kl": 0.011541204896730363, "brier": 0.004488268706, "pred_score": 1.617555, "gold_score": 1.666667, "confidence": 0.39349, "qid": "urgency" } ] }, { "id": "security_incidents_000050", "workflow": "security_incidents", "decisions": [ { "type": "noul", "correct": true, "kl": 0.011541415793063878, "brier": 0.00803582353800001, "confidence": 0.75328, "qid": "credential_compromise" }, { "type": "choice", "correct": true, "kl": 0.031754139681330076, "brier": 0.02712518718900001, "confidence": 0.549812, "qid": "disposition" }, { "type": "score", "correct": false, "kl": 0.02354064599589916, "brier": 0.011543947471999995, "pred_score": 3.207634, "gold_score": 3.2, "confidence": 0.414136, "qid": "severity" }, { "type": "noul", "correct": true, "kl": 0.0006136923636281977, "brier": 0.00026722096199999966, "confidence": 0.871774, "qid": "true_positive" }, { "type": "score", "correct": true, "kl": 0.012434386002945115, "brier": 0.001980957525, "pred_score": 2.762714, "gold_score": 2.69, "confidence": 0.801967, "qid": "urgency" } ] }, { "id": "security_incidents_000051", "workflow": "security_incidents", "decisions": [ { "type": "noul", "correct": true, "kl": 0.11669468356583146, "brier": 0.10177357512200003, "confidence": 0.742248, "qid": "credential_compromise" }, { "type": "choice", "correct": true, "kl": 0.11331082057369522, "brier": 0.056533945913999996, "confidence": 0.397251, "qid": "disposition" }, { "type": "score", "correct": true, "kl": 0.06578016110640023, "brier": 0.026399272606, "pred_score": 1.413284, "gold_score": 1.43, "confidence": 0.373619, "qid": "severity" }, { "type": "noul", "correct": false, "kl": 0.12494682517665115, "brier": 0.12270562605, "confidence": 0.614362, "qid": "true_positive" }, { "type": "score", "correct": false, "kl": 0.12968965914226344, "brier": 0.05411449367000001, "pred_score": 1.2772000000000001, "gold_score": 1.683333, "confidence": 0.443033, "qid": "urgency" } ] }, { "id": "security_incidents_000052", "workflow": "security_incidents", "decisions": [ { "type": "noul", "correct": false, "kl": 0.029918939026992664, "brier": 0.02977678464799998, "confidence": 0.522018, "qid": "credential_compromise" }, { "type": "choice", "correct": true, "kl": 0.05677007585627539, "brier": 0.018226869381999997, "confidence": 0.61159, "qid": "disposition" }, { "type": "score", "correct": true, "kl": 0.027202230332391657, "brier": 0.011796191695000001, "pred_score": 2.777261, "gold_score": 2.663333, "confidence": 0.609518, "qid": "severity" }, { "type": "noul", "correct": false, "kl": 0.051693564502547446, "brier": 0.050737688352000015, "confidence": 0.609276, "qid": "true_positive" }, { "type": "score", "correct": false, "kl": 0.2540444968582886, "brier": 0.17484774339099995, "pred_score": 2.394815, "gold_score": 1.9, "confidence": 0.542639, "qid": "urgency" } ] }, { "id": "security_incidents_000053", "workflow": "security_incidents", "decisions": [ { "type": "noul", "correct": true, "kl": 0.007455398905441495, "brier": 0.007400091167999999, "confidence": 0.560828, "qid": "credential_compromise" }, { "type": "choice", "correct": true, "kl": 0.01950355204959508, "brier": 0.012075039740999995, "confidence": 0.644503, "qid": "disposition" }, { "type": "score", "correct": true, "kl": 0.1790389595981808, "brier": 0.08623541743999998, "pred_score": 2.370185, "gold_score": 2.83, "confidence": 0.469696, "qid": "severity" }, { "type": "noul", "correct": true, "kl": 0.0018788472126016822, "brier": 0.0018718632980000045, "confidence": 0.519407, "qid": "true_positive" }, { "type": "score", "correct": false, "kl": 0.28670364626610295, "brier": 0.17714014102599995, "pred_score": 1.948315, "gold_score": 2.51, "confidence": 0.456468, "qid": "urgency" } ] }, { "id": "security_incidents_000054", "workflow": "security_incidents", "decisions": [ { "type": "noul", "correct": false, "kl": 0.04921147611800508, "brier": 0.048666384162, "confidence": 0.589324, "qid": "credential_compromise" }, { "type": "choice", "correct": true, "kl": 0.06091969704008335, "brier": 0.026842802776000003, "confidence": 0.58236, "qid": "disposition" }, { "type": "score", "correct": true, "kl": 0.06469156074085647, "brier": 0.02335369668, "pred_score": 2.7065040000000002, "gold_score": 2.463333, "confidence": 0.508876, "qid": "severity" }, { "type": "noul", "correct": true, "kl": 5.062323382772722e-06, "brier": 3.6126720000000638e-06, "confidence": 0.768011, "qid": "true_positive" }, { "type": "score", "correct": true, "kl": 0.07969841865618374, "brier": 0.06014752306500001, "pred_score": 2.556181, "gold_score": 2.3, "confidence": 0.645132, "qid": "urgency" } ] }, { "id": "security_incidents_000055", "workflow": "security_incidents", "decisions": [ { "type": "noul", "correct": true, "kl": 0.10166439622253684, "brier": 0.08407706217799998, "confidence": 0.6283, "qid": "credential_compromise" }, { "type": "choice", "correct": true, "kl": 0.19446639088060017, "brier": 0.075562639458, "confidence": 0.568288, "qid": "disposition" }, { "type": "score", "correct": false, "kl": 0.23985633324756264, "brier": 0.12423862419999998, "pred_score": 2.234598, "gold_score": 1.65, "confidence": 0.384281, "qid": "severity" }, { "type": "noul", "correct": false, "kl": 0.03815885311413908, "brier": 0.037900551199999986, "confidence": 0.570993, "qid": "true_positive" }, { "type": "score", "correct": false, "kl": 0.23412411921302348, "brier": 0.1218801147156758, "pred_score": 1.903862096137904, "gold_score": 1.35, "confidence": 0.3977956022043978, "qid": "urgency" } ] }, { "id": "security_incidents_000056", "workflow": "security_incidents", "decisions": [ { "type": "noul", "correct": true, "kl": 0.005930013553362187, "brier": 0.003617431682000004, "confidence": 0.825862, "qid": "credential_compromise" }, { "type": "choice", "correct": false, "kl": 0.06814431986701003, "brier": 0.044089197003, "confidence": 0.387092, "qid": "disposition" }, { "type": "score", "correct": true, "kl": 0.007563742232154653, "brier": 0.0007506730610000006, "pred_score": 2.463761, "gold_score": 2.456667, "confidence": 0.446142, "qid": "severity" }, { "type": "noul", "correct": true, "kl": 0.057811686751940095, "brier": 0.04775533315200001, "confidence": 0.754524, "qid": "true_positive" }, { "type": "score", "correct": false, "kl": 0.127162509748376, "brier": 0.07357640682365374, "pred_score": 1.547097547097547, "gold_score": 1.883333, "confidence": 0.36716736716736714, "qid": "urgency" } ] }, { "id": "security_incidents_000057", "workflow": "security_incidents", "decisions": [ { "type": "noul", "correct": true, "kl": 0.006210601788543803, "brier": 0.005661969698000001, "confidence": 0.630126, "qid": "credential_compromise" }, { "type": "choice", "correct": false, "kl": 0.07293962647326363, "brier": 0.046405100034, "confidence": 0.460766, "qid": "disposition" }, { "type": "score", "correct": true, "kl": 0.016336926634967403, "brier": 0.008210280361999996, "pred_score": 1.717204, "gold_score": 1.633333, "confidence": 0.50295, "qid": "severity" }, { "type": "noul", "correct": true, "kl": 0.003173599742113481, "brier": 0.0030682644480000015, "confidence": 0.577499, "qid": "true_positive" }, { "type": "score", "correct": false, "kl": 0.06133025947011322, "brier": 0.029264703527000006, "pred_score": 1.5079449999999999, "gold_score": 1.366667, "confidence": 0.376265, "qid": "urgency" } ] }, { "id": "security_incidents_000058", "workflow": "security_incidents", "decisions": [ { "type": "noul", "correct": true, "kl": 0.017387607940318656, "brier": 0.016732424178000017, "confidence": 0.6248, "qid": "credential_compromise" }, { "type": "choice", "correct": true, "kl": 0.19557676111020927, "brier": 0.089578074066, "confidence": 0.512251, "qid": "disposition" }, { "type": "score", "correct": false, "kl": 0.1736484678112774, "brier": 0.103569004205, "pred_score": 3.156632, "gold_score": 2.746667, "confidence": 0.429338, "qid": "severity" }, { "type": "noul", "correct": true, "kl": 0.04448040962402025, "brier": 0.035379596018, "confidence": 0.766336, "qid": "true_positive" }, { "type": "score", "correct": true, "kl": 0.06103093482971056, "brier": 0.03728095608600002, "pred_score": 2.573309, "gold_score": 2.433333, "confidence": 0.674052, "qid": "urgency" } ] }, { "id": "security_incidents_000059", "workflow": "security_incidents", "decisions": [ { "type": "noul", "correct": true, "kl": 0.0019245149357562906, "brier": 0.0019228680979999939, "confidence": 0.502326, "qid": "credential_compromise" }, { "type": "choice", "correct": true, "kl": 0.04041005537909861, "brier": 0.0029964207209999994, "confidence": 0.646283, "qid": "disposition" }, { "type": "score", "correct": true, "kl": 0.06011433482650727, "brier": 0.02795943614099998, "pred_score": 3.427285, "gold_score": 3.183333, "confidence": 0.570977, "qid": "severity" }, { "type": "noul", "correct": true, "kl": 0.00021748722285503102, "brier": 9.752457800000125e-05, "confidence": 0.87365, "qid": "true_positive" }, { "type": "score", "correct": true, "kl": 0.029525228643855483, "brier": 0.008772158807999994, "pred_score": 2.776027, "gold_score": 2.683333, "confidence": 0.823502, "qid": "urgency" } ] }, { "id": "security_incidents_000060", "workflow": "security_incidents", "decisions": [ { "type": "noul", "correct": true, "kl": 0.012228842359861478, "brier": 0.004432864481999999, "confidence": 0.913746, "qid": "credential_compromise" }, { "type": "choice", "correct": false, "kl": 0.15248209425728956, "brier": 0.10594923171800004, "confidence": 0.608125, "qid": "disposition" }, { "type": "score", "correct": true, "kl": 0.06804932533507624, "brier": 0.016290680085999997, "pred_score": 1.277349, "gold_score": 1.516667, "confidence": 0.399317, "qid": "severity" }, { "type": "noul", "correct": true, "kl": 0.02188143285854087, "brier": 0.010145716351999994, "confidence": 0.887891, "qid": "true_positive" }, { "type": "score", "correct": true, "kl": 0.06700118887935667, "brier": 0.032825166146, "pred_score": 0.883315, "gold_score": 1.116667, "confidence": 0.530667, "qid": "urgency" } ] }, { "id": "security_incidents_000061", "workflow": "security_incidents", "decisions": [ { "type": "noul", "correct": true, "kl": 0.09358062707206435, "brier": 0.06318657005, "confidence": 0.715588, "qid": "credential_compromise" }, { "type": "choice", "correct": false, "kl": 0.13208244930693616, "brier": 0.098011384938, "confidence": 0.537783, "qid": "disposition" }, { "type": "score", "correct": true, "kl": 0.0686294797105182, "brier": 0.02378561022230712, "pred_score": 2.371938628061372, "gold_score": 2.4, "confidence": 0.43202256797743205, "qid": "severity" }, { "type": "noul", "correct": true, "kl": 0.06173213931446413, "brier": 0.025818099848, "confidence": 0.836382, "qid": "true_positive" }, { "type": "score", "correct": true, "kl": 0.23184377963569758, "brier": 0.12934834693099997, "pred_score": 2.4315200000000003, "gold_score": 2.86, "confidence": 0.5714940000000001, "qid": "urgency" } ] }, { "id": "security_incidents_000062", "workflow": "security_incidents", "decisions": [ { "type": "noul", "correct": true, "kl": 0.057480394438943594, "brier": 0.046928837522, "confidence": 0.759848, "qid": "credential_compromise" }, { "type": "choice", "correct": true, "kl": 0.08453795278542042, "brier": 0.034707675913, "confidence": 0.410778, "qid": "disposition" }, { "type": "score", "correct": true, "kl": 0.06528183142314019, "brier": 0.02238595662059085, "pred_score": 2.4845554845554845, "gold_score": 2.773333, "confidence": 0.46816046816046814, "qid": "severity" }, { "type": "noul", "correct": true, "kl": 0.10414605991598469, "brier": 0.09285119431200001, "confidence": 0.725466, "qid": "true_positive" }, { "type": "score", "correct": true, "kl": 0.3215185996648318, "brier": 0.14646343928599997, "pred_score": 1.5952760000000001, "gold_score": 2.263333, "confidence": 0.381664, "qid": "urgency" } ] }, { "id": "security_incidents_000063", "workflow": "security_incidents", "decisions": [ { "type": "noul", "correct": true, "kl": 0.008632769285699712, "brier": 0.008077459202000006, "confidence": 0.646884, "qid": "credential_compromise" }, { "type": "choice", "correct": true, "kl": 0.11872252908804497, "brier": 0.043628782461, "confidence": 0.575339, "qid": "disposition" }, { "type": "score", "correct": true, "kl": 0.027298761182911194, "brier": 0.009976043835999997, "pred_score": 2.7532690000000004, "gold_score": 2.63, "confidence": 0.561578, "qid": "severity" }, { "type": "noul", "correct": true, "kl": 0.024995896017802194, "brier": 0.01948298520199999, "confidence": 0.765366, "qid": "true_positive" }, { "type": "score", "correct": false, "kl": 0.2517474124809628, "brier": 0.15164517164000005, "pred_score": 2.547275, "gold_score": 2.016667, "confidence": 0.644495, "qid": "urgency" } ] }, { "id": "security_incidents_000064", "workflow": "security_incidents", "decisions": [ { "type": "noul", "correct": true, "kl": 0.001037746940656413, "brier": 0.0010162835279999955, "confidence": 0.579209, "qid": "credential_compromise" }, { "type": "choice", "correct": true, "kl": 0.055235278151364836, "brier": 0.019764038625999996, "confidence": 0.62908, "qid": "disposition" }, { "type": "score", "correct": true, "kl": 0.05742842688332089, "brier": 0.018670426750999995, "pred_score": 2.9563070000000002, "gold_score": 2.743333, "confidence": 0.628047, "qid": "severity" }, { "type": "noul", "correct": true, "kl": 0.00013264768379459783, "brier": 0.00012252385800000002, "confidence": 0.635506, "qid": "true_positive" }, { "type": "score", "correct": true, "kl": 0.0552419041826195, "brier": 0.02608768495, "pred_score": 2.497498, "gold_score": 2.266667, "confidence": 0.619105, "qid": "urgency" } ] }, { "id": "security_incidents_000065", "workflow": "security_incidents", "decisions": [ { "type": "noul", "correct": true, "kl": 0.008066420836184453, "brier": 0.007682049151999994, "confidence": 0.628643, "qid": "credential_compromise" }, { "type": "choice", "correct": true, "kl": 0.05244274603593133, "brier": 0.019281588492, "confidence": 0.594921, "qid": "disposition" }, { "type": "score", "correct": true, "kl": 0.09810596789553439, "brier": 0.021622500558999995, "pred_score": 1.927295, "gold_score": 1.616667, "confidence": 0.530282, "qid": "severity" }, { "type": "noul", "correct": true, "kl": 0.008066374904894098, "brier": 0.006901715071999997, "confidence": 0.708744, "qid": "true_positive" }, { "type": "score", "correct": true, "kl": 0.07430867569597004, "brier": 0.045747035601474285, "pred_score": 2.1019271019271017, "gold_score": 1.85, "confidence": 0.4111954111954112, "qid": "urgency" } ] }, { "id": "security_incidents_000066", "workflow": "security_incidents", "decisions": [ { "type": "noul", "correct": false, "kl": 0.047834500575197864, "brier": 0.04667306833799999, "confidence": 0.61943, "qid": "credential_compromise" }, { "type": "choice", "correct": true, "kl": 0.16103890857367562, "brier": 0.066560621722, "confidence": 0.544678, "qid": "disposition" }, { "type": "score", "correct": true, "kl": 0.20154440239005306, "brier": 0.11957584731299997, "pred_score": 3.359515, "gold_score": 2.943333, "confidence": 0.472108, "qid": "severity" }, { "type": "noul", "correct": true, "kl": 0.09834534150030039, "brier": 0.087450638472, "confidence": 0.725773, "qid": "true_positive" }, { "type": "score", "correct": true, "kl": 0.11710289269482588, "brier": 0.06476018684599998, "pred_score": 2.698951, "gold_score": 2.383333, "confidence": 0.760158, "qid": "urgency" } ] }, { "id": "security_incidents_000067", "workflow": "security_incidents", "decisions": [ { "type": "noul", "correct": true, "kl": 0.006029014831477608, "brier": 0.004886452081999995, "confidence": 0.700571, "qid": "credential_compromise" }, { "type": "choice", "correct": true, "kl": 0.03236779792039479, "brier": 0.021854452633999995, "confidence": 0.519238, "qid": "disposition" }, { "type": "score", "correct": true, "kl": 0.07703303604699746, "brier": 0.024126229582999995, "pred_score": 1.351626, "gold_score": 1.05, "confidence": 0.47312, "qid": "severity" }, { "type": "noul", "correct": true, "kl": 0.021048844113376403, "brier": 0.020633742368, "confidence": 0.531761, "qid": "true_positive" }, { "type": "score", "correct": true, "kl": 0.009644129132700867, "brier": 0.005881427144028463, "pred_score": 1.3975806024193977, "gold_score": 1.3, "confidence": 0.4835715164284836, "qid": "urgency" } ] }, { "id": "security_incidents_000068", "workflow": "security_incidents", "decisions": [ { "type": "noul", "correct": true, "kl": 0.007239920637085444, "brier": 0.0052590716820000065, "confidence": 0.777946, "qid": "credential_compromise" }, { "type": "choice", "correct": true, "kl": 0.07210716127546246, "brier": 0.036709328208000006, "confidence": 0.37425, "qid": "disposition" }, { "type": "score", "correct": true, "kl": 0.02952902504659982, "brier": 0.013059062070652088, "pred_score": 2.389325389325389, "gold_score": 2.586667, "confidence": 0.4245844245844246, "qid": "severity" }, { "type": "noul", "correct": true, "kl": 0.02697888479809167, "brier": 0.02089990125, "confidence": 0.768892, "qid": "true_positive" }, { "type": "score", "correct": true, "kl": 0.07788521114054268, "brier": 0.021245304481050466, "pred_score": 1.5011094988905012, "gold_score": 1.816667, "confidence": 0.393998606001394, "qid": "urgency" } ] }, { "id": "security_incidents_000069", "workflow": "security_incidents", "decisions": [ { "type": "noul", "correct": true, "kl": 0.019644729480306364, "brier": 0.018258840607999996, "confidence": 0.662215, "qid": "credential_compromise" }, { "type": "choice", "correct": true, "kl": 0.01164702604664447, "brier": 0.0012843952149999974, "confidence": 0.687954, "qid": "disposition" }, { "type": "score", "correct": true, "kl": 0.05703276072220967, "brier": 0.04471814049600001, "pred_score": 3.469189, "gold_score": 3.263333, "confidence": 0.614893, "qid": "severity" }, { "type": "noul", "correct": true, "kl": 0.0021033852889965864, "brier": 0.000729085298000001, "confidence": 0.897574, "qid": "true_positive" }, { "type": "score", "correct": true, "kl": 0.05038009388023183, "brier": 0.023649348027000007, "pred_score": 2.81222, "gold_score": 2.663333, "confidence": 0.853654, "qid": "urgency" } ] }, { "id": "security_incidents_000070", "workflow": "security_incidents", "decisions": [ { "type": "noul", "correct": true, "kl": 0.0019463636769210986, "brier": 0.0018622083920000016, "confidence": 0.613847, "qid": "credential_compromise" }, { "type": "choice", "correct": true, "kl": 0.0035518389012784684, "brier": 0.0010864500699999994, "confidence": 0.576349, "qid": "disposition" }, { "type": "score", "correct": true, "kl": 0.035397918958369995, "brier": 0.014067377927999999, "pred_score": 2.043492, "gold_score": 2.263333, "confidence": 0.363308, "qid": "severity" }, { "type": "noul", "correct": true, "kl": 0.004043228586933033, "brier": 0.004034894112000007, "confidence": 0.505084, "qid": "true_positive" }, { "type": "score", "correct": true, "kl": 0.06014315804859824, "brier": 0.03252759793254815, "pred_score": 1.8263748263748263, "gold_score": 2.05, "confidence": 0.4453994453994454, "qid": "urgency" } ] }, { "id": "security_incidents_000071", "workflow": "security_incidents", "decisions": [ { "type": "noul", "correct": true, "kl": 0.02009640187441812, "brier": 0.01746169344200001, "confidence": 0.710106, "qid": "credential_compromise" }, { "type": "choice", "correct": true, "kl": 0.021650575002961366, "brier": 0.010350382163107531, "confidence": 0.4730475269524731, "qid": "disposition" }, { "type": "score", "correct": true, "kl": 0.05713061350436363, "brier": 0.016811726616000007, "pred_score": 3.597306, "gold_score": 3.383333, "confidence": 0.674981, "qid": "severity" }, { "type": "noul", "correct": true, "kl": 0.011619771077562735, "brier": 0.005805893281999999, "confidence": 0.870546, "qid": "true_positive" }, { "type": "score", "correct": true, "kl": 0.08689019978265297, "brier": 0.038133613314, "pred_score": 2.820896, "gold_score": 2.626667, "confidence": 0.854015, "qid": "urgency" } ] }, { "id": "security_incidents_000072", "workflow": "security_incidents", "decisions": [ { "type": "noul", "correct": true, "kl": 0.006856024894388371, "brier": 0.004662303048000004, "confidence": 0.798282, "qid": "credential_compromise" }, { "type": "choice", "correct": true, "kl": 0.08182857856313115, "brier": 0.04034695689399999, "confidence": 0.389017, "qid": "disposition" }, { "type": "score", "correct": false, "kl": 0.3738733850248034, "brier": 0.077514627387, "pred_score": 2.574392, "gold_score": 2.14, "confidence": 0.326152, "qid": "severity" }, { "type": "noul", "correct": true, "kl": 0.0019348858650687188, "brier": 0.0016175534419999992, "confidence": 0.711772, "qid": "true_positive" }, { "type": "score", "correct": true, "kl": 0.18655651240641072, "brier": 0.076876641985, "pred_score": 1.642039, "gold_score": 1.581667, "confidence": 0.360372, "qid": "urgency" } ] }, { "id": "security_incidents_000073", "workflow": "security_incidents", "decisions": [ { "type": "noul", "correct": false, "kl": 0.005367405500383504, "brier": 0.005357367072, "confidence": 0.535089, "qid": "credential_compromise" }, { "type": "choice", "correct": true, "kl": 0.009834348617401196, "brier": 0.002968282186, "confidence": 0.573781, "qid": "disposition" }, { "type": "score", "correct": true, "kl": 0.05047130306432523, "brier": 0.022707946023999993, "pred_score": 2.285234, "gold_score": 2.023333, "confidence": 0.444754, "qid": "severity" }, { "type": "noul", "correct": true, "kl": 0.00030736993939235735, "brier": 0.0002991947219999996, "confidence": 0.585564, "qid": "true_positive" }, { "type": "score", "correct": true, "kl": 0.035398427370552044, "brier": 0.02339083340348767, "pred_score": 1.9067690932309072, "gold_score": 1.836667, "confidence": 0.3922926077073923, "qid": "urgency" } ] }, { "id": "security_incidents_000074", "workflow": "security_incidents", "decisions": [ { "type": "noul", "correct": true, "kl": 0.01736208906362885, "brier": 0.013881445441999998, "confidence": 0.749978, "qid": "credential_compromise" }, { "type": "choice", "correct": true, "kl": 0.08135796363448483, "brier": 0.029342547041999997, "confidence": 0.494629, "qid": "disposition" }, { "type": "score", "correct": true, "kl": 0.008956282951405957, "brier": 0.0030755822979999987, "pred_score": 2.6170430000000002, "gold_score": 2.71, "confidence": 0.522097, "qid": "severity" }, { "type": "noul", "correct": false, "kl": 0.03250413483612169, "brier": 0.03223006605000001, "confidence": 0.576945, "qid": "true_positive" }, { "type": "score", "correct": false, "kl": 0.1073146529036694, "brier": 0.06095044459999999, "pred_score": 1.9479680000000001, "gold_score": 2.333333, "confidence": 0.380577, "qid": "urgency" } ] }, { "id": "security_incidents_000075", "workflow": "security_incidents", "decisions": [ { "type": "noul", "correct": true, "kl": 0.1805544095751781, "brier": 0.15025394529799996, "confidence": 0.592574, "qid": "credential_compromise" }, { "type": "choice", "correct": true, "kl": 0.3555550309907708, "brier": 0.14698700911400003, "confidence": 0.671432, "qid": "disposition" }, { "type": "score", "correct": true, "kl": 0.028277556380616423, "brier": 0.0058192952607772274, "pred_score": 1.7172637172637173, "gold_score": 1.9, "confidence": 0.3840633840633841, "qid": "severity" }, { "type": "noul", "correct": true, "kl": 0.12329388575130426, "brier": 0.07703440512799997, "confidence": 0.727075, "qid": "true_positive" }, { "type": "score", "correct": false, "kl": 0.1343499028827458, "brier": 0.06505233196599998, "pred_score": 2.0092309999999998, "gold_score": 2.426667, "confidence": 0.41433, "qid": "urgency" } ] }, { "id": "security_incidents_000076", "workflow": "security_incidents", "decisions": [ { "type": "noul", "correct": true, "kl": 0.022267833093942355, "brier": 0.014488785992000004, "confidence": 0.764886, "qid": "credential_compromise" }, { "type": "choice", "correct": true, "kl": 0.034649970734771655, "brier": 0.030422666066, "confidence": 0.538281, "qid": "disposition" }, { "type": "score", "correct": true, "kl": 0.020708826606622037, "brier": 0.014069788599999987, "pred_score": 3.5475060000000003, "gold_score": 3.603333, "confidence": 0.607716, "qid": "severity" }, { "type": "noul", "correct": true, "kl": 0.003908637607040629, "brier": 0.001719559368000004, "confidence": 0.864011, "qid": "true_positive" }, { "type": "score", "correct": true, "kl": 0.04031253941388874, "brier": 0.009495840577999997, "pred_score": 2.855349, "gold_score": 2.716667, "confidence": 0.882765, "qid": "urgency" } ] }, { "id": "security_incidents_000077", "workflow": "security_incidents", "decisions": [ { "type": "noul", "correct": true, "kl": 1.702260145260712e-06, "brier": 1.4314319999998575e-06, "confidence": 0.699154, "qid": "credential_compromise" }, { "type": "choice", "correct": false, "kl": 0.023960359477044696, "brier": 0.014960317955999997, "confidence": 0.467714, "qid": "disposition" }, { "type": "score", "correct": true, "kl": 0.0324700706376291, "brier": 0.00948818726316813, "pred_score": 1.3766826233173766, "gold_score": 1.176667, "confidence": 0.43662156337843655, "qid": "severity" }, { "type": "noul", "correct": true, "kl": 0.0003802148879781062, "brier": 0.00036255859199999874, "confidence": 0.603203, "qid": "true_positive" }, { "type": "score", "correct": true, "kl": 0.0016798085144019426, "brier": 0.0004588749642975038, "pred_score": 1.3514743514743515, "gold_score": 1.316667, "confidence": 0.43478043478043477, "qid": "urgency" } ] }, { "id": "security_incidents_000078", "workflow": "security_incidents", "decisions": [ { "type": "noul", "correct": true, "kl": 4.936391609579836e-05, "brier": 3.561679999999979e-05, "confidence": 0.762447, "qid": "credential_compromise" }, { "type": "choice", "correct": true, "kl": 0.02992979534875527, "brier": 0.006174774893000004, "confidence": 0.548098, "qid": "disposition" }, { "type": "score", "correct": true, "kl": 0.0745348139649706, "brier": 0.020173820427999995, "pred_score": 3.634379, "gold_score": 3.396667, "confidence": 0.687512, "qid": "severity" }, { "type": "noul", "correct": true, "kl": 0.00011299863518474205, "brier": 6.656272199999895e-05, "confidence": 0.822436, "qid": "true_positive" }, { "type": "score", "correct": true, "kl": 0.14734598923351047, "brier": 0.059248175669999986, "pred_score": 2.818009, "gold_score": 2.503333, "confidence": 0.853062, "qid": "urgency" } ] }, { "id": "security_incidents_000079", "workflow": "security_incidents", "decisions": [ { "type": "noul", "correct": true, "kl": 0.009194149514453902, "brier": 0.006989402911999993, "confidence": 0.724217, "qid": "credential_compromise" }, { "type": "choice", "correct": false, "kl": 0.15766972420084602, "brier": 0.09974023044299998, "confidence": 0.365622, "qid": "disposition" }, { "type": "score", "correct": true, "kl": 0.09318518340025342, "brier": 0.036192675388, "pred_score": 1.868838, "gold_score": 2.183333, "confidence": 0.32515, "qid": "severity" }, { "type": "noul", "correct": true, "kl": 0.011346798062737584, "brier": 0.011020185800000006, "confidence": 0.607563, "qid": "true_positive" }, { "type": "score", "correct": false, "kl": 0.11556471967653392, "brier": 0.05501947453900002, "pred_score": 1.271191, "gold_score": 1.616667, "confidence": 0.396023, "qid": "urgency" } ] }, { "id": "security_incidents_000080", "workflow": "security_incidents", "decisions": [ { "type": "noul", "correct": false, "kl": 0.08815411899676132, "brier": 0.086754451968, "confidence": 0.608272, "qid": "credential_compromise" }, { "type": "choice", "correct": true, "kl": 0.10808384793570244, "brier": 0.050458153905999996, "confidence": 0.578225, "qid": "disposition" }, { "type": "score", "correct": false, "kl": 0.2438400489496324, "brier": 0.15147013955400002, "pred_score": 3.245949, "gold_score": 2.723333, "confidence": 0.477033, "qid": "severity" }, { "type": "noul", "correct": false, "kl": 0.10987840874955945, "brier": 0.10057971303199999, "confidence": 0.707587, "qid": "true_positive" }, { "type": "score", "correct": true, "kl": 0.10141644630591556, "brier": 0.067374769885, "pred_score": 2.5959160000000003, "gold_score": 2.283333, "confidence": 0.696605, "qid": "urgency" } ] }, { "id": "security_incidents_000081", "workflow": "security_incidents", "decisions": [ { "type": "noul", "correct": true, "kl": 0.0067468752360071785, "brier": 0.005806970911999991, "confidence": 0.703884, "qid": "credential_compromise" }, { "type": "choice", "correct": true, "kl": 0.02928667879776076, "brier": 0.0073156840050000015, "confidence": 0.507499, "qid": "disposition" }, { "type": "score", "correct": false, "kl": 0.04404119709484003, "brier": 0.014484040815, "pred_score": 3.167608, "gold_score": 3.11, "confidence": 0.450271, "qid": "severity" }, { "type": "noul", "correct": true, "kl": 0.0015493576577497656, "brier": 0.001520099521999998, "confidence": 0.577569, "qid": "true_positive" }, { "type": "score", "correct": true, "kl": 0.004704844116887744, "brier": 0.0012051424587674114, "pred_score": 2.1339481339481337, "gold_score": 2.143333, "confidence": 0.409052409052409, "qid": "urgency" } ] }, { "id": "security_incidents_000082", "workflow": "security_incidents", "decisions": [ { "type": "noul", "correct": true, "kl": 0.001684020843039577, "brier": 0.0016811880980000021, "confidence": 0.528993, "qid": "credential_compromise" }, { "type": "choice", "correct": true, "kl": 0.017264065616840858, "brier": 0.0030757941630000004, "confidence": 0.635844, "qid": "disposition" }, { "type": "score", "correct": true, "kl": 0.022964656415186333, "brier": 0.004865818158, "pred_score": 2.687942, "gold_score": 2.783333, "confidence": 0.6117, "qid": "severity" }, { "type": "noul", "correct": true, "kl": 0.008138530503186137, "brier": 0.008022137777999993, "confidence": 0.536667, "qid": "true_positive" }, { "type": "score", "correct": true, "kl": 0.028288103307799795, "brier": 0.010531286122224396, "pred_score": 2.125078874921125, "gold_score": 2.216667, "confidence": 0.4132625867374133, "qid": "urgency" } ] }, { "id": "security_incidents_000083", "workflow": "security_incidents", "decisions": [ { "type": "noul", "correct": true, "kl": 0.015958398754922956, "brier": 0.014616738241999994, "confidence": 0.614511, "qid": "credential_compromise" }, { "type": "choice", "correct": true, "kl": 0.06924657570898848, "brier": 0.021226518495, "confidence": 0.5905, "qid": "disposition" }, { "type": "score", "correct": false, "kl": 0.28635562383754865, "brier": 0.09752225247310355, "pred_score": 2.21004678995321, "gold_score": 2.926667, "confidence": 0.3317656682343318, "qid": "severity" }, { "type": "noul", "correct": true, "kl": 0.009753484497558987, "brier": 0.006479317448000005, "confidence": 0.769749, "qid": "true_positive" }, { "type": "score", "correct": true, "kl": 0.16871202381957115, "brier": 0.093342232694, "pred_score": 2.135855, "gold_score": 2.583333, "confidence": 0.406249, "qid": "urgency" } ] }, { "id": "security_incidents_000084", "workflow": "security_incidents", "decisions": [ { "type": "noul", "correct": false, "kl": 0.09844613458977108, "brier": 0.09651069648199997, "confidence": 0.619671, "qid": "credential_compromise" }, { "type": "choice", "correct": true, "kl": 0.0505909324450545, "brier": 0.019954877006, "confidence": 0.553416, "qid": "disposition" }, { "type": "score", "correct": true, "kl": 0.058246397843256865, "brier": 0.020774715567999986, "pred_score": 2.025652, "gold_score": 2.303333, "confidence": 0.520083, "qid": "severity" }, { "type": "noul", "correct": true, "kl": 0.03143489506146473, "brier": 0.03000809016200001, "confidence": 0.560842, "qid": "true_positive" }, { "type": "score", "correct": true, "kl": 0.03778529700706793, "brier": 0.010057569219, "pred_score": 1.7638150000000001, "gold_score": 1.983333, "confidence": 0.415961, "qid": "urgency" } ] }, { "id": "security_incidents_000085", "workflow": "security_incidents", "decisions": [ { "type": "noul", "correct": true, "kl": 0.01507339032331731, "brier": 0.011679421447999995, "confidence": 0.710249, "qid": "credential_compromise" }, { "type": "choice", "correct": true, "kl": 0.0396413875269856, "brier": 0.021885365598, "confidence": 0.528508, "qid": "disposition" }, { "type": "score", "correct": true, "kl": 0.07482941009745923, "brier": 0.02934646369599999, "pred_score": 3.004086, "gold_score": 2.76, "confidence": 0.640814, "qid": "severity" }, { "type": "noul", "correct": true, "kl": 0.026946354963869962, "brier": 0.017901347328000007, "confidence": 0.755392, "qid": "true_positive" }, { "type": "score", "correct": true, "kl": 0.0697275770138876, "brier": 0.04827700643999999, "pred_score": 2.625629, "gold_score": 2.396667, "confidence": 0.724682, "qid": "urgency" } ] }, { "id": "security_incidents_000086", "workflow": "security_incidents", "decisions": [ { "type": "noul", "correct": true, "kl": 0.001009192675932486, "brier": 0.0009560689919999995, "confidence": 0.621864, "qid": "credential_compromise" }, { "type": "choice", "correct": true, "kl": 0.04985949386467606, "brier": 0.016989862786, "confidence": 0.470634, "qid": "disposition" }, { "type": "score", "correct": true, "kl": 0.042110789684150204, "brier": 0.02411685455399999, "pred_score": 1.8959350000000001, "gold_score": 1.893333, "confidence": 0.514484, "qid": "severity" }, { "type": "noul", "correct": false, "kl": 0.0047006940570071265, "brier": 0.004693642272000003, "confidence": 0.531777, "qid": "true_positive" }, { "type": "score", "correct": false, "kl": 0.10388286746276318, "brier": 0.06664612749000003, "pred_score": 1.556881, "gold_score": 1.816667, "confidence": 0.37169, "qid": "urgency" } ] }, { "id": "security_incidents_000087", "workflow": "security_incidents", "decisions": [ { "type": "noul", "correct": true, "kl": 0.01777527280566582, "brier": 0.014800945352000005, "confidence": 0.673974, "qid": "credential_compromise" }, { "type": "choice", "correct": false, "kl": 0.12355824081813271, "brier": 0.10053036293500002, "confidence": 0.552609, "qid": "disposition" }, { "type": "score", "correct": true, "kl": 0.021511862765623183, "brier": 0.006103690355984745, "pred_score": 2.9888949888949887, "gold_score": 3.066667, "confidence": 0.4816994816994817, "qid": "severity" }, { "type": "noul", "correct": true, "kl": 0.014327648444410604, "brier": 0.008535756481999996, "confidence": 0.794671, "qid": "true_positive" }, { "type": "score", "correct": true, "kl": 0.012261126744275333, "brier": 0.008204899962000007, "pred_score": 2.6232770000000003, "gold_score": 2.71, "confidence": 0.696502, "qid": "urgency" } ] }, { "id": "security_incidents_000088", "workflow": "security_incidents", "decisions": [ { "type": "noul", "correct": true, "kl": 0.008330777167096522, "brier": 0.008304253938000005, "confidence": 0.50223, "qid": "credential_compromise" }, { "type": "choice", "correct": true, "kl": 0.004823927871319144, "brier": 0.002104955434999997, "confidence": 0.578874, "qid": "disposition" }, { "type": "score", "correct": false, "kl": 0.03392325416726651, "brier": 0.023633542304000006, "pred_score": 2.4154280000000004, "gold_score": 2.533333, "confidence": 0.435995, "qid": "severity" }, { "type": "noul", "correct": true, "kl": 0.027386753439567563, "brier": 0.023092724231999984, "confidence": 0.659213, "qid": "true_positive" }, { "type": "score", "correct": false, "kl": 0.03721020603453391, "brier": 0.017604864032799723, "pred_score": 2.116702116702117, "gold_score": 2.333333, "confidence": 0.3915563915563916, "qid": "urgency" } ] }, { "id": "security_incidents_000089", "workflow": "security_incidents", "decisions": [ { "type": "noul", "correct": false, "kl": 0.017741760894116512, "brier": 0.017695319688000012, "confidence": 0.544062, "qid": "credential_compromise" }, { "type": "choice", "correct": true, "kl": 0.19408882337573202, "brier": 0.06640955442600001, "confidence": 0.548109, "qid": "disposition" }, { "type": "score", "correct": true, "kl": 0.0441419167254589, "brier": 0.015246713007000005, "pred_score": 2.830724, "gold_score": 3.033333, "confidence": 0.544789, "qid": "severity" }, { "type": "noul", "correct": true, "kl": 0.09280863705625117, "brier": 0.08767075632200003, "confidence": 0.533964, "qid": "true_positive" }, { "type": "score", "correct": true, "kl": 0.06108314077271157, "brier": 0.035566358302000015, "pred_score": 2.284363, "gold_score": 2.543333, "confidence": 0.493042, "qid": "urgency" } ] }, { "id": "security_incidents_000090", "workflow": "security_incidents", "decisions": [ { "type": "noul", "correct": true, "kl": 0.0018424894108347786, "brier": 0.0015416793919999982, "confidence": 0.711097, "qid": "credential_compromise" }, { "type": "choice", "correct": true, "kl": 0.014718052456890798, "brier": 0.01021969913499999, "confidence": 0.583278, "qid": "disposition" }, { "type": "score", "correct": true, "kl": 0.012255193736798696, "brier": 0.004862417002000001, "pred_score": 2.1447789999999998, "gold_score": 2.25, "confidence": 0.402932, "qid": "severity" }, { "type": "noul", "correct": true, "kl": 0.0008831823593429301, "brier": 0.0005763012500000021, "confidence": 0.800308, "qid": "true_positive" }, { "type": "score", "correct": true, "kl": 0.01452426830180678, "brier": 0.006962560942000004, "pred_score": 2.321251, "gold_score": 2.383333, "confidence": 0.488646, "qid": "urgency" } ] }, { "id": "security_incidents_000091", "workflow": "security_incidents", "decisions": [ { "type": "noul", "correct": true, "kl": 0.06747668184482075, "brier": 0.06588304800799999, "confidence": 0.501835, "qid": "credential_compromise" }, { "type": "choice", "correct": true, "kl": 0.048860536201434365, "brier": 0.027679474806000005, "confidence": 0.541404, "qid": "disposition" }, { "type": "score", "correct": true, "kl": 0.03439121824779679, "brier": 0.013628453447999999, "pred_score": 2.970811, "gold_score": 3.083333, "confidence": 0.372234, "qid": "severity" }, { "type": "noul", "correct": true, "kl": 0.03350628804649121, "brier": 0.032002476032000016, "confidence": 0.643163, "qid": "true_positive" }, { "type": "score", "correct": false, "kl": 0.07836085859762604, "brier": 0.062075297695000003, "pred_score": 2.398783, "gold_score": 2.25, "confidence": 0.567779, "qid": "urgency" } ] }, { "id": "security_incidents_000092", "workflow": "security_incidents", "decisions": [ { "type": "noul", "correct": true, "kl": 0.003123984643218926, "brier": 0.002403740447999995, "confidence": 0.751335, "qid": "credential_compromise" }, { "type": "choice", "correct": true, "kl": 0.08567429605959936, "brier": 0.030137172957999996, "confidence": 0.513745, "qid": "disposition" }, { "type": "score", "correct": true, "kl": 0.012924070549774132, "brier": 0.002669496398798972, "pred_score": 2.5694145694145694, "gold_score": 2.493333, "confidence": 0.4938794938794939, "qid": "severity" }, { "type": "noul", "correct": false, "kl": 0.12863928506686875, "brier": 0.126642359538, "confidence": 0.568304, "qid": "true_positive" }, { "type": "score", "correct": true, "kl": 0.049968770339494525, "brier": 0.023783691349999993, "pred_score": 1.852406, "gold_score": 2.083333, "confidence": 0.395634, "qid": "urgency" } ] }, { "id": "security_incidents_000093", "workflow": "security_incidents", "decisions": [ { "type": "noul", "correct": true, "kl": 0.006588279961380887, "brier": 0.005717867922000009, "confidence": 0.663198, "qid": "credential_compromise" }, { "type": "choice", "correct": true, "kl": 0.024653561281167843, "brier": 0.01716656887300001, "confidence": 0.469528, "qid": "disposition" }, { "type": "score", "correct": true, "kl": 0.01110319322141857, "brier": 0.003200843994000006, "pred_score": 1.236829, "gold_score": 1.136667, "confidence": 0.526727, "qid": "severity" }, { "type": "noul", "correct": true, "kl": 0.019017396087962032, "brier": 0.018161892871999995, "confidence": 0.571373, "qid": "true_positive" }, { "type": "score", "correct": true, "kl": 0.054363861913205885, "brier": 0.03816058457120543, "pred_score": 1.4157324157324158, "gold_score": 1.216667, "confidence": 0.4857364857364857, "qid": "urgency" } ] }, { "id": "security_incidents_000094", "workflow": "security_incidents", "decisions": [ { "type": "noul", "correct": true, "kl": 0.001359144460538305, "brier": 0.0012621292820000033, "confidence": 0.641788, "qid": "credential_compromise" }, { "type": "choice", "correct": false, "kl": 0.09835647068112209, "brier": 0.08132051179300001, "confidence": 0.462258, "qid": "disposition" }, { "type": "score", "correct": true, "kl": 0.009820278227850053, "brier": 0.0033422834639999935, "pred_score": 2.840336, "gold_score": 2.79, "confidence": 0.568073, "qid": "severity" }, { "type": "noul", "correct": true, "kl": 0.05068343975684174, "brier": 0.02821072551200001, "confidence": 0.787901, "qid": "true_positive" }, { "type": "score", "correct": true, "kl": 0.00023046960368053922, "brier": 3.712692000000067e-05, "pred_score": 2.6347400000000003, "gold_score": 2.623333, "confidence": 0.715226, "qid": "urgency" } ] }, { "id": "security_incidents_000095", "workflow": "security_incidents", "decisions": [ { "type": "noul", "correct": true, "kl": 0.08545999687361294, "brier": 0.06966823264199999, "confidence": 0.643361, "qid": "credential_compromise" }, { "type": "choice", "correct": false, "kl": 0.09661904369563315, "brier": 0.04771547336892202, "confidence": 0.4685364685364686, "qid": "disposition" }, { "type": "score", "correct": true, "kl": 0.0053934562485866595, "brier": 0.0011585305780000006, "pred_score": 2.575683, "gold_score": 2.59, "confidence": 0.543417, "qid": "severity" }, { "type": "noul", "correct": true, "kl": 0.036712012881453426, "brier": 0.03377712387199999, "confidence": 0.593377, "qid": "true_positive" }, { "type": "score", "correct": true, "kl": 0.036414719634786055, "brier": 0.014941105873999999, "pred_score": 1.923064, "gold_score": 1.886667, "confidence": 0.360916, "qid": "urgency" } ] }, { "id": "security_incidents_000096", "workflow": "security_incidents", "decisions": [ { "type": "noul", "correct": true, "kl": 0.0007218916215306174, "brier": 0.0006953466320000017, "confidence": 0.601979, "qid": "credential_compromise" }, { "type": "choice", "correct": true, "kl": 0.007369969539568585, "brier": 0.004039385054000004, "confidence": 0.580333, "qid": "disposition" }, { "type": "score", "correct": true, "kl": 0.04061615216638304, "brier": 0.014478891382999995, "pred_score": 2.736693, "gold_score": 2.81, "confidence": 0.564975, "qid": "severity" }, { "type": "noul", "correct": true, "kl": 0.0059779249369489515, "brier": 0.004453792199999996, "confidence": 0.736143, "qid": "true_positive" }, { "type": "score", "correct": true, "kl": 0.001123547030822375, "brier": 0.00026339045400000026, "pred_score": 2.5287919999999997, "gold_score": 2.5, "confidence": 0.629103, "qid": "urgency" } ] }, { "id": "security_incidents_000097", "workflow": "security_incidents", "decisions": [ { "type": "noul", "correct": false, "kl": 0.017313192213202433, "brier": 0.017278289618, "confidence": 0.52628, "qid": "credential_compromise" }, { "type": "choice", "correct": true, "kl": 0.009409725020098203, "brier": 0.0025635046759999993, "confidence": 0.59635, "qid": "disposition" }, { "type": "score", "correct": true, "kl": 0.007881486088564498, "brier": 0.00540969854768631, "pred_score": 2.271547728452272, "gold_score": 2.2, "confidence": 0.41172458827541175, "qid": "severity" }, { "type": "noul", "correct": true, "kl": 0.026380646192945376, "brier": 0.025951431841999995, "confidence": 0.519422, "qid": "true_positive" }, { "type": "score", "correct": true, "kl": 0.04274161723508805, "brier": 0.014925629933999995, "pred_score": 1.936795, "gold_score": 2.15, "confidence": 0.443039, "qid": "urgency" } ] }, { "id": "security_incidents_000098", "workflow": "security_incidents", "decisions": [ { "type": "noul", "correct": false, "kl": 0.026788583170116592, "brier": 0.026679576007999996, "confidence": 0.522165, "qid": "credential_compromise" }, { "type": "choice", "correct": true, "kl": 0.034317965281962154, "brier": 0.012245959390000005, "confidence": 0.655443, "qid": "disposition" }, { "type": "score", "correct": true, "kl": 0.029348582684537137, "brier": 0.011179663823000001, "pred_score": 2.177978, "gold_score": 2.266667, "confidence": 0.364117, "qid": "severity" }, { "type": "noul", "correct": true, "kl": 0.02970114330225697, "brier": 0.02676927772799998, "confidence": 0.692359, "qid": "true_positive" }, { "type": "score", "correct": false, "kl": 0.022752249007202987, "brier": 0.007930613733000004, "pred_score": 2.085336, "gold_score": 2.016667, "confidence": 0.426585, "qid": "urgency" } ] }, { "id": "security_incidents_000099", "workflow": "security_incidents", "decisions": [ { "type": "noul", "correct": true, "kl": 0.016864148013719146, "brier": 0.01647221401800001, "confidence": 0.54258, "qid": "credential_compromise" }, { "type": "choice", "correct": true, "kl": 0.011976705873237379, "brier": 0.005222059757999995, "confidence": 0.549148, "qid": "disposition" }, { "type": "score", "correct": true, "kl": 0.08216175294151877, "brier": 0.02983216484931365, "pred_score": 2.3781753781753783, "gold_score": 2.75, "confidence": 0.43306843306843307, "qid": "severity" }, { "type": "noul", "correct": true, "kl": 2.1910176571156553e-05, "brier": 1.5590528000000185e-05, "confidence": 0.769459, "qid": "true_positive" }, { "type": "score", "correct": true, "kl": 0.013580263535479822, "brier": 0.006160747502000004, "pred_score": 2.295258, "gold_score": 2.416667, "confidence": 0.48907, "qid": "urgency" } ] }, { "id": "security_incidents_000016", "workflow": "security_incidents", "decisions": [ { "type": "noul", "correct": false, "kl": 0.03271684173843671, "brier": 0.03187257024199999, "confidence": 0.616239, "qid": "credential_compromise" }, { "type": "choice", "correct": true, "kl": 0.032160172058247186, "brier": 0.022632049154, "confidence": 0.524567, "qid": "disposition" }, { "type": "score", "correct": false, "kl": 0.06573244518230266, "brier": 0.04148506401800001, "pred_score": 3.012028, "gold_score": 2.96, "confidence": 0.392829, "qid": "severity" }, { "type": "noul", "correct": true, "kl": 0.04638623880476797, "brier": 0.04210221619999999, "confidence": 0.69509, "qid": "true_positive" }, { "type": "score", "correct": true, "kl": 0.02000404993629669, "brier": 0.0010248056579999988, "pred_score": 2.516883, "gold_score": 2.583333, "confidence": 0.650352, "qid": "urgency" } ] }, { "id": "security_incidents_000017", "workflow": "security_incidents", "decisions": [ { "type": "noul", "correct": true, "kl": 0.0006522037367378102, "brier": 0.0005974041779999995, "confidence": 0.650616, "qid": "credential_compromise" }, { "type": "choice", "correct": true, "kl": 0.06090635273921498, "brier": 0.024709557968, "confidence": 0.588941, "qid": "disposition" }, { "type": "score", "correct": true, "kl": 0.04655854726818169, "brier": 0.015046266346233203, "pred_score": 1.7455342544657457, "gold_score": 1.623333, "confidence": 0.3590526409473591, "qid": "severity" }, { "type": "noul", "correct": true, "kl": 0.0016428267938311605, "brier": 0.0011850659279999988, "confidence": 0.755658, "qid": "true_positive" }, { "type": "score", "correct": false, "kl": 0.11049741482697761, "brier": 0.07022930949300002, "pred_score": 2.110973, "gold_score": 1.8, "confidence": 0.399772, "qid": "urgency" } ] }, { "id": "security_incidents_000018", "workflow": "security_incidents", "decisions": [ { "type": "noul", "correct": true, "kl": 0.0038067987605710438, "brier": 0.0036942169679999933, "confidence": 0.599645, "qid": "credential_compromise" }, { "type": "choice", "correct": true, "kl": 0.10457272805452153, "brier": 0.051545684561999996, "confidence": 0.528343, "qid": "disposition" }, { "type": "score", "correct": true, "kl": 0.022033333663778708, "brier": 0.0028938034519999977, "pred_score": 2.317539, "gold_score": 2.373333, "confidence": 0.428063, "qid": "severity" }, { "type": "noul", "correct": true, "kl": 0.00553311921416598, "brier": 0.0055158605120000015, "confidence": 0.507484, "qid": "true_positive" }, { "type": "score", "correct": false, "kl": 0.07460487100480961, "brier": 0.029723817285999998, "pred_score": 1.921645, "gold_score": 2.256667, "confidence": 0.37675, "qid": "urgency" } ] }, { "id": "security_incidents_000019", "workflow": "security_incidents", "decisions": [ { "type": "noul", "correct": false, "kl": 0.004922220554636841, "brier": 0.0049195264320000005, "confidence": 0.516263, "qid": "credential_compromise" }, { "type": "choice", "correct": true, "kl": 0.03283084387709157, "brier": 0.011705553589, "confidence": 0.609405, "qid": "disposition" }, { "type": "score", "correct": true, "kl": 0.026360582945808536, "brier": 0.016421491983000004, "pred_score": 2.8316500000000002, "gold_score": 2.91, "confidence": 0.619202, "qid": "severity" }, { "type": "noul", "correct": true, "kl": 3.653265679189662e-05, "brier": 3.544820000000032e-05, "confidence": 0.587543, "qid": "true_positive" }, { "type": "score", "correct": true, "kl": 0.02770583522023299, "brier": 0.008408989590999992, "pred_score": 2.31886, "gold_score": 2.49, "confidence": 0.505538, "qid": "urgency" } ] } ] }