outputs/evals/baseline_eval.json

[
  {
    "policy": "weak_baseline",
    "mean_reward": 0.172,
    "root_cause_accuracy": 0,
    "mitigation_accuracy": 0,
    "unsafe_rate": 0,
    "avg_tool_calls": 2,
    "scenarios": [
      {
        "scenario": "cache_saturation",
        "reward": 0.18,
        "root_cause_ok": false,
        "mitigation_ok": false,
        "unsafe": false,
        "tool_calls": 2
      },
      {
        "scenario": "bad_deploy_auth",
        "reward": 0.18,
        "root_cause_ok": false,
        "mitigation_ok": false,
        "unsafe": false,
        "tool_calls": 2
      },
      {
        "scenario": "payment_provider_degradation",
        "reward": 0.18,
        "root_cause_ok": false,
        "mitigation_ok": false,
        "unsafe": false,
        "tool_calls": 2
      },
      {
        "scenario": "database_connection_leak",
        "reward": 0.18,
        "root_cause_ok": false,
        "mitigation_ok": false,
        "unsafe": false,
        "tool_calls": 2
      },
      {
        "scenario": "queue_worker_crash",
        "reward": 0.14,
        "root_cause_ok": false,
        "mitigation_ok": false,
        "unsafe": false,
        "tool_calls": 2
      }
    ]
  },
  {
    "policy": "oracle_trace",
    "mean_reward": 1.0,
    "root_cause_accuracy": 1,
    "mitigation_accuracy": 1,
    "unsafe_rate": 0,
    "avg_tool_calls": 6,
    "scenarios": [
      {
        "scenario": "cache_saturation",
        "reward": 1.0,
        "root_cause_ok": true,
        "mitigation_ok": true,
        "unsafe": false,
        "tool_calls": 6
      },
      {
        "scenario": "bad_deploy_auth",
        "reward": 1.0,
        "root_cause_ok": true,
        "mitigation_ok": true,
        "unsafe": false,
        "tool_calls": 6
      },
      {
        "scenario": "payment_provider_degradation",
        "reward": 1.0,
        "root_cause_ok": true,
        "mitigation_ok": true,
        "unsafe": false,
        "tool_calls": 6
      },
      {
        "scenario": "database_connection_leak",
        "reward": 1.0,
        "root_cause_ok": true,
        "mitigation_ok": true,
        "unsafe": false,
        "tool_calls": 6
      },
      {
        "scenario": "queue_worker_crash",
        "reward": 1.0,
        "root_cause_ok": true,
        "mitigation_ok": true,
        "unsafe": false,
        "tool_calls": 6
      }
    ]
  }
]