| { |
| "dataset": "LocalLLaMA/typed-decisions", |
| "split": "test", |
| "selected_cases": 400, |
| "successful_cases": 400, |
| "failed_cases": 0, |
| "scored_cases": 400, |
| "overall": { |
| "decisions": 2000, |
| "correct": 1652, |
| "accuracy": 0.826, |
| "kl": 0.0803057449071361, |
| "brier": 0.04295575756754636, |
| "ece_15": 0.15975580646719287 |
| }, |
| "metric_notes": { |
| "accuracy": "argmax prediction against the gold argmax label", |
| "kl": "mean KL(gold || prediction), natural logarithm", |
| "brier": "mean sum over classes of (prediction - soft gold)^2", |
| "ece_15": "15 equal-width bins using top probability and hard argmax correctness" |
| }, |
| "latency_ms": { |
| "p50": 106.56309803016484, |
| "p95": 135.15076198382303 |
| }, |
| "scoring_errors": [], |
| "by_workflow": { |
| "agent_trace_observability": { |
| "decisions": 500, |
| "correct": 389, |
| "accuracy": 0.778, |
| "kl": 0.08059402466036933, |
| "brier": 0.0463716026092487, |
| "ece_15": 0.185638753939392 |
| }, |
| "customer_service": { |
| "decisions": 500, |
| "correct": 425, |
| "accuracy": 0.85, |
| "kl": 0.08023071358764519, |
| "brier": 0.04071135055898889, |
| "ece_15": 0.15487085697191613 |
| }, |
| "invoice_processing": { |
| "decisions": 500, |
| "correct": 429, |
| "accuracy": 0.858, |
| "kl": 0.09564322592504344, |
| "brier": 0.05099897025345711, |
| "ece_15": 0.09557565599617864 |
| }, |
| "security_incidents": { |
| "decisions": 500, |
| "correct": 409, |
| "accuracy": 0.818, |
| "kl": 0.06475501545548643, |
| "brier": 0.033741106848490764, |
| "ece_15": 0.20823487398337517 |
| } |
| }, |
| "by_type": { |
| "choice": { |
| "decisions": 600, |
| "correct": 478, |
| "accuracy": 0.7966666666666666, |
| "kl": 0.10667343256268746, |
| "brier": 0.05479604004707588, |
| "ece_15": 0.15765588434266006 |
| }, |
| "noul": { |
| "decisions": 600, |
| "correct": 527, |
| "accuracy": 0.8783333333333333, |
| "kl": 0.04097212613378347, |
| "brier": 0.030015148967631963, |
| "ece_15": 0.10459395721244316 |
| }, |
| "score": { |
| "decisions": 800, |
| "correct": 647, |
| "accuracy": 0.80875, |
| "kl": 0.09003019324548704, |
| "brier": 0.04378100215783503, |
| "ece_15": 0.20270213500165474 |
| } |
| }, |
| "by_question": { |
| "agent_trace_observability/action": { |
| "decisions": 100, |
| "correct": 72, |
| "accuracy": 0.72, |
| "kl": 0.08790775035944622, |
| "brier": 0.045996883503236996, |
| "ece_15": 0.1923458341441178 |
| }, |
| "agent_trace_observability/needs_review": { |
| "decisions": 100, |
| "correct": 90, |
| "accuracy": 0.9, |
| "kl": 0.033611987181820735, |
| "brier": 0.02631921507244329, |
| "ece_15": 0.17988898418843746 |
| }, |
| "agent_trace_observability/outcome": { |
| "decisions": 100, |
| "correct": 75, |
| "accuracy": 0.75, |
| "kl": 0.10884891853683683, |
| "brier": 0.06128768621134422, |
| "ece_15": 0.22472407297158692 |
| }, |
| "agent_trace_observability/risk": { |
| "decisions": 100, |
| "correct": 82, |
| "accuracy": 0.82, |
| "kl": 0.08616333697498625, |
| "brier": 0.051944564727536655, |
| "ece_15": 0.21548420564294987 |
| }, |
| "agent_trace_observability/urgency": { |
| "decisions": 100, |
| "correct": 70, |
| "accuracy": 0.7, |
| "kl": 0.0864381302487566, |
| "brier": 0.04630966353168234, |
| "ece_15": 0.17894596456783354 |
| }, |
| "customer_service/action": { |
| "decisions": 100, |
| "correct": 72, |
| "accuracy": 0.72, |
| "kl": 0.16485836116812386, |
| "brier": 0.08073784085652039, |
| "ece_15": 0.1081227785754724 |
| }, |
| "customer_service/category": { |
| "decisions": 100, |
| "correct": 98, |
| "accuracy": 0.98, |
| "kl": 0.04316980921110677, |
| "brier": 0.015287821774841741, |
| "ece_15": 0.12546654650846617 |
| }, |
| "customer_service/churn_risk": { |
| "decisions": 100, |
| "correct": 89, |
| "accuracy": 0.89, |
| "kl": 0.06087967761779349, |
| "brier": 0.02739846473868238, |
| "ece_15": 0.24890314749550854 |
| }, |
| "customer_service/needs_human": { |
| "decisions": 100, |
| "correct": 82, |
| "accuracy": 0.82, |
| "kl": 0.05379561473975391, |
| "brier": 0.0447702723712042, |
| "ece_15": 0.10668263003230095 |
| }, |
| "customer_service/urgency": { |
| "decisions": 100, |
| "correct": 84, |
| "accuracy": 0.84, |
| "kl": 0.07845010520144792, |
| "brier": 0.035362353053695735, |
| "ece_15": 0.23016179276997029 |
| }, |
| "invoice_processing/discrepancy_severity": { |
| "decisions": 100, |
| "correct": 86, |
| "accuracy": 0.86, |
| "kl": 0.16211923361755753, |
| "brier": 0.07577871556224147, |
| "ece_15": 0.11399502083683016 |
| }, |
| "invoice_processing/disposition": { |
| "decisions": 100, |
| "correct": 81, |
| "accuracy": 0.81, |
| "kl": 0.14687131365910766, |
| "brier": 0.08027354501411901, |
| "ece_15": 0.14137618761078863 |
| }, |
| "invoice_processing/duplicate": { |
| "decisions": 100, |
| "correct": 91, |
| "accuracy": 0.91, |
| "kl": 0.03952439425291494, |
| "brier": 0.026807131545151866, |
| "ece_15": 0.0636753043998033 |
| }, |
| "invoice_processing/matches_order": { |
| "decisions": 100, |
| "correct": 95, |
| "accuracy": 0.95, |
| "kl": 0.05999561126647583, |
| "brier": 0.03594590921788439, |
| "ece_15": 0.060309361563995516 |
| }, |
| "invoice_processing/urgency": { |
| "decisions": 100, |
| "correct": 76, |
| "accuracy": 0.76, |
| "kl": 0.06970557682916122, |
| "brier": 0.036189549927888774, |
| "ece_15": 0.17601861818019318 |
| }, |
| "security_incidents/credential_compromise": { |
| "decisions": 100, |
| "correct": 82, |
| "accuracy": 0.82, |
| "kl": 0.031176862771824396, |
| "brier": 0.025240369597254987, |
| "ece_15": 0.14930089343339203 |
| }, |
| "security_incidents/disposition": { |
| "decisions": 100, |
| "correct": 80, |
| "accuracy": 0.8, |
| "kl": 0.0883844424415034, |
| "brier": 0.04519246292239289, |
| "ece_15": 0.2498926800447473 |
| }, |
| "security_incidents/severity": { |
| "decisions": 100, |
| "correct": 84, |
| "accuracy": 0.84, |
| "kl": 0.07871596952034915, |
| "brier": 0.0302813399315595, |
| "ece_15": 0.3170699022538294 |
| }, |
| "security_incidents/true_positive": { |
| "decisions": 100, |
| "correct": 87, |
| "accuracy": 0.87, |
| "kl": 0.027728286589911, |
| "brier": 0.02100799600185304, |
| "ece_15": 0.19495224751532075 |
| }, |
| "security_incidents/urgency": { |
| "decisions": 100, |
| "correct": 76, |
| "accuracy": 0.76, |
| "kl": 0.09776951595384421, |
| "brier": 0.0469833657893934, |
| "ece_15": 0.2065293719543915 |
| } |
| } |
| } |
|
|
