data/benchmark-results.json

{
  "scope": "synthetic deterministic smoke benchmark",
  "sessions": [
    {
      "session_id": "synthetic-01",
      "expected": [],
      "predicted": [],
      "pass": true
    },
    {
      "session_id": "synthetic-02",
      "expected": [
        "scope_violation"
      ],
      "predicted": [
        "scope_violation"
      ],
      "pass": true
    },
    {
      "session_id": "synthetic-03",
      "expected": [
        "approval_bypass"
      ],
      "predicted": [
        "approval_bypass"
      ],
      "pass": true
    },
    {
      "session_id": "synthetic-04",
      "expected": [
        "prompt_injection",
        "scope_violation",
        "secret_exposure"
      ],
      "predicted": [
        "prompt_injection",
        "scope_violation",
        "secret_exposure"
      ],
      "pass": true
    },
    {
      "session_id": "synthetic-05",
      "expected": [
        "runaway_loop"
      ],
      "predicted": [
        "runaway_loop"
      ],
      "pass": true
    },
    {
      "session_id": "synthetic-06",
      "expected": [
        "silent_tool_failure"
      ],
      "predicted": [
        "silent_tool_failure"
      ],
      "pass": true
    },
    {
      "session_id": "synthetic-07",
      "expected": [
        "secret_exposure"
      ],
      "predicted": [
        "secret_exposure"
      ],
      "pass": true
    },
    {
      "session_id": "synthetic-08",
      "expected": [
        "scope_violation"
      ],
      "predicted": [
        "scope_violation"
      ],
      "pass": true
    },
    {
      "session_id": "synthetic-09",
      "expected": [],
      "predicted": [],
      "pass": true
    },
    {
      "session_id": "synthetic-10",
      "expected": [
        "scope_violation"
      ],
      "predicted": [
        "scope_violation"
      ],
      "pass": true
    },
    {
      "session_id": "synthetic-11",
      "expected": [
        "approval_bypass"
      ],
      "predicted": [
        "approval_bypass"
      ],
      "pass": true
    },
    {
      "session_id": "synthetic-12",
      "expected": [
        "prompt_injection",
        "scope_violation",
        "secret_exposure"
      ],
      "predicted": [
        "prompt_injection",
        "scope_violation",
        "secret_exposure"
      ],
      "pass": true
    },
    {
      "session_id": "synthetic-13",
      "expected": [
        "runaway_loop"
      ],
      "predicted": [
        "runaway_loop"
      ],
      "pass": true
    },
    {
      "session_id": "synthetic-14",
      "expected": [
        "silent_tool_failure"
      ],
      "predicted": [
        "silent_tool_failure"
      ],
      "pass": true
    },
    {
      "session_id": "synthetic-15",
      "expected": [
        "secret_exposure"
      ],
      "predicted": [
        "secret_exposure"
      ],
      "pass": true
    },
    {
      "session_id": "synthetic-16",
      "expected": [
        "scope_violation"
      ],
      "predicted": [
        "scope_violation"
      ],
      "pass": true
    },
    {
      "session_id": "synthetic-17",
      "expected": [],
      "predicted": [],
      "pass": true
    },
    {
      "session_id": "synthetic-18",
      "expected": [
        "approval_bypass"
      ],
      "predicted": [
        "approval_bypass"
      ],
      "pass": true
    },
    {
      "session_id": "synthetic-19",
      "expected": [
        "scope_violation"
      ],
      "predicted": [
        "scope_violation"
      ],
      "pass": true
    },
    {
      "session_id": "synthetic-20",
      "expected": [],
      "predicted": [],
      "pass": true
    }
  ],
  "true_positives": 20,
  "false_positives": 0,
  "false_negatives": 0,
  "precision": 1.0,
  "recall": 1.0,
  "f1": 1.0,
  "per_category": {
    "approval_bypass": {
      "tp": 3,
      "fp": 0,
      "fn": 0,
      "tn": 17,
      "precision": 1.0,
      "recall": 1.0,
      "f1": 1.0
    },
    "none": {
      "tp": 0,
      "fp": 0,
      "fn": 0,
      "tn": 20,
      "precision": 1.0,
      "recall": 1.0,
      "f1": 1.0
    },
    "prompt_injection": {
      "tp": 2,
      "fp": 0,
      "fn": 0,
      "tn": 18,
      "precision": 1.0,
      "recall": 1.0,
      "f1": 1.0
    },
    "runaway_loop": {
      "tp": 2,
      "fp": 0,
      "fn": 0,
      "tn": 18,
      "precision": 1.0,
      "recall": 1.0,
      "f1": 1.0
    },
    "scope_violation": {
      "tp": 7,
      "fp": 0,
      "fn": 0,
      "tn": 13,
      "precision": 1.0,
      "recall": 1.0,
      "f1": 1.0
    },
    "secret_exposure": {
      "tp": 4,
      "fp": 0,
      "fn": 0,
      "tn": 16,
      "precision": 1.0,
      "recall": 1.0,
      "f1": 1.0
    },
    "silent_tool_failure": {
      "tp": 2,
      "fp": 0,
      "fn": 0,
      "tn": 18,
      "precision": 1.0,
      "recall": 1.0,
      "f1": 1.0
    }
  },
  "confusion": {
    "approval_bypass": {
      "tp": 3,
      "fp": 0,
      "fn": 0,
      "tn": 17
    },
    "none": {
      "tp": 0,
      "fp": 0,
      "fn": 0,
      "tn": 20
    },
    "prompt_injection": {
      "tp": 2,
      "fp": 0,
      "fn": 0,
      "tn": 18
    },
    "runaway_loop": {
      "tp": 2,
      "fp": 0,
      "fn": 0,
      "tn": 18
    },
    "scope_violation": {
      "tp": 7,
      "fp": 0,
      "fn": 0,
      "tn": 13
    },
    "secret_exposure": {
      "tp": 4,
      "fp": 0,
      "fn": 0,
      "tn": 16
    },
    "silent_tool_failure": {
      "tp": 2,
      "fp": 0,
      "fn": 0,
      "tn": 18
    }
  },
  "warning": "Synthetic fixtures validate implemented detector behavior only; they are not evidence of real-world accuracy."
}