{
  "dataset": "LocalLLaMA/typed-decisions",
  "split": "test",
  "selected_cases": 400,
  "successful_cases": 400,
  "failed_cases": 0,
  "scored_cases": 400,
  "overall": {
    "decisions": 2000,
    "correct": 1652,
    "accuracy": 0.826,
    "kl": 0.0803057449071361,
    "brier": 0.04295575756754636,
    "ece_15": 0.15975580646719287
  },
  "metric_notes": {
    "accuracy": "argmax prediction against the gold argmax label",
    "kl": "mean KL(gold || prediction), natural logarithm",
    "brier": "mean sum over classes of (prediction - soft gold)^2",
    "ece_15": "15 equal-width bins using top probability and hard argmax correctness"
  },
  "latency_ms": {
    "p50": 106.56309803016484,
    "p95": 135.15076198382303
  },
  "scoring_errors": [],
  "by_workflow": {
    "agent_trace_observability": {
      "decisions": 500,
      "correct": 389,
      "accuracy": 0.778,
      "kl": 0.08059402466036933,
      "brier": 0.0463716026092487,
      "ece_15": 0.185638753939392
    },
    "customer_service": {
      "decisions": 500,
      "correct": 425,
      "accuracy": 0.85,
      "kl": 0.08023071358764519,
      "brier": 0.04071135055898889,
      "ece_15": 0.15487085697191613
    },
    "invoice_processing": {
      "decisions": 500,
      "correct": 429,
      "accuracy": 0.858,
      "kl": 0.09564322592504344,
      "brier": 0.05099897025345711,
      "ece_15": 0.09557565599617864
    },
    "security_incidents": {
      "decisions": 500,
      "correct": 409,
      "accuracy": 0.818,
      "kl": 0.06475501545548643,
      "brier": 0.033741106848490764,
      "ece_15": 0.20823487398337517
    }
  },
  "by_type": {
    "choice": {
      "decisions": 600,
      "correct": 478,
      "accuracy": 0.7966666666666666,
      "kl": 0.10667343256268746,
      "brier": 0.05479604004707588,
      "ece_15": 0.15765588434266006
    },
    "noul": {
      "decisions": 600,
      "correct": 527,
      "accuracy": 0.8783333333333333,
      "kl": 0.04097212613378347,
      "brier": 0.030015148967631963,
      "ece_15": 0.10459395721244316
    },
    "score": {
      "decisions": 800,
      "correct": 647,
      "accuracy": 0.80875,
      "kl": 0.09003019324548704,
      "brier": 0.04378100215783503,
      "ece_15": 0.20270213500165474
    }
  },
  "by_question": {
    "agent_trace_observability/action": {
      "decisions": 100,
      "correct": 72,
      "accuracy": 0.72,
      "kl": 0.08790775035944622,
      "brier": 0.045996883503236996,
      "ece_15": 0.1923458341441178
    },
    "agent_trace_observability/needs_review": {
      "decisions": 100,
      "correct": 90,
      "accuracy": 0.9,
      "kl": 0.033611987181820735,
      "brier": 0.02631921507244329,
      "ece_15": 0.17988898418843746
    },
    "agent_trace_observability/outcome": {
      "decisions": 100,
      "correct": 75,
      "accuracy": 0.75,
      "kl": 0.10884891853683683,
      "brier": 0.06128768621134422,
      "ece_15": 0.22472407297158692
    },
    "agent_trace_observability/risk": {
      "decisions": 100,
      "correct": 82,
      "accuracy": 0.82,
      "kl": 0.08616333697498625,
      "brier": 0.051944564727536655,
      "ece_15": 0.21548420564294987
    },
    "agent_trace_observability/urgency": {
      "decisions": 100,
      "correct": 70,
      "accuracy": 0.7,
      "kl": 0.0864381302487566,
      "brier": 0.04630966353168234,
      "ece_15": 0.17894596456783354
    },
    "customer_service/action": {
      "decisions": 100,
      "correct": 72,
      "accuracy": 0.72,
      "kl": 0.16485836116812386,
      "brier": 0.08073784085652039,
      "ece_15": 0.1081227785754724
    },
    "customer_service/category": {
      "decisions": 100,
      "correct": 98,
      "accuracy": 0.98,
      "kl": 0.04316980921110677,
      "brier": 0.015287821774841741,
      "ece_15": 0.12546654650846617
    },
    "customer_service/churn_risk": {
      "decisions": 100,
      "correct": 89,
      "accuracy": 0.89,
      "kl": 0.06087967761779349,
      "brier": 0.02739846473868238,
      "ece_15": 0.24890314749550854
    },
    "customer_service/needs_human": {
      "decisions": 100,
      "correct": 82,
      "accuracy": 0.82,
      "kl": 0.05379561473975391,
      "brier": 0.0447702723712042,
      "ece_15": 0.10668263003230095
    },
    "customer_service/urgency": {
      "decisions": 100,
      "correct": 84,
      "accuracy": 0.84,
      "kl": 0.07845010520144792,
      "brier": 0.035362353053695735,
      "ece_15": 0.23016179276997029
    },
    "invoice_processing/discrepancy_severity": {
      "decisions": 100,
      "correct": 86,
      "accuracy": 0.86,
      "kl": 0.16211923361755753,
      "brier": 0.07577871556224147,
      "ece_15": 0.11399502083683016
    },
    "invoice_processing/disposition": {
      "decisions": 100,
      "correct": 81,
      "accuracy": 0.81,
      "kl": 0.14687131365910766,
      "brier": 0.08027354501411901,
      "ece_15": 0.14137618761078863
    },
    "invoice_processing/duplicate": {
      "decisions": 100,
      "correct": 91,
      "accuracy": 0.91,
      "kl": 0.03952439425291494,
      "brier": 0.026807131545151866,
      "ece_15": 0.0636753043998033
    },
    "invoice_processing/matches_order": {
      "decisions": 100,
      "correct": 95,
      "accuracy": 0.95,
      "kl": 0.05999561126647583,
      "brier": 0.03594590921788439,
      "ece_15": 0.060309361563995516
    },
    "invoice_processing/urgency": {
      "decisions": 100,
      "correct": 76,
      "accuracy": 0.76,
      "kl": 0.06970557682916122,
      "brier": 0.036189549927888774,
      "ece_15": 0.17601861818019318
    },
    "security_incidents/credential_compromise": {
      "decisions": 100,
      "correct": 82,
      "accuracy": 0.82,
      "kl": 0.031176862771824396,
      "brier": 0.025240369597254987,
      "ece_15": 0.14930089343339203
    },
    "security_incidents/disposition": {
      "decisions": 100,
      "correct": 80,
      "accuracy": 0.8,
      "kl": 0.0883844424415034,
      "brier": 0.04519246292239289,
      "ece_15": 0.2498926800447473
    },
    "security_incidents/severity": {
      "decisions": 100,
      "correct": 84,
      "accuracy": 0.84,
      "kl": 0.07871596952034915,
      "brier": 0.0302813399315595,
      "ece_15": 0.3170699022538294
    },
    "security_incidents/true_positive": {
      "decisions": 100,
      "correct": 87,
      "accuracy": 0.87,
      "kl": 0.027728286589911,
      "brier": 0.02100799600185304,
      "ece_15": 0.19495224751532075
    },
    "security_incidents/urgency": {
      "decisions": 100,
      "correct": 76,
      "accuracy": 0.76,
      "kl": 0.09776951595384421,
      "brier": 0.0469833657893934,
      "ece_15": 0.2065293719543915
    }
  }
}