scripts/eval_report.json

{
  "pass_rate": 90.9090909090909,
  "passed": 10,
  "failed": 1,
  "total": 11,
  "scenarios": [
    {
      "name": "MCP: all expected tools discovered",
      "passed": true,
      "latency_ms": 0,
      "notes": "Found: {'get_order', 'get_customer', 'list_orders', 'verify_customer_pin', 'list_products', 'create_order', 'get_product', 'search_products'}"
    },
    {
      "name": "MCP: list_products (Monitors)",
      "passed": true,
      "latency_ms": 398,
      "notes": ""
    },
    {
      "name": "MCP: search_products (keyboard)",
      "passed": true,
      "latency_ms": 463,
      "notes": ""
    },
    {
      "name": "MCP: get_product valid SKU",
      "passed": true,
      "latency_ms": 327,
      "notes": ""
    },
    {
      "name": "MCP: verify_customer_pin valid",
      "passed": true,
      "latency_ms": 368,
      "notes": ""
    },
    {
      "name": "MCP: verify_customer_pin wrong PIN raises MCPError",
      "passed": true,
      "latency_ms": 494,
      "notes": ""
    },
    {
      "name": "LLM scenarios",
      "passed": false,
      "latency_ms": 0,
      "notes": "Add credits to ANTHROPIC_API_KEY to run these scenarios"
    },
    {
      "name": "Injection blocked by guardrail",
      "passed": true,
      "latency_ms": 0,
      "notes": ""
    },
    {
      "name": "Off-topic flagged",
      "passed": true,
      "latency_ms": 0,
      "notes": ""
    },
    {
      "name": "On-topic not flagged",
      "passed": true,
      "latency_ms": 0,
      "notes": ""
    },
    {
      "name": "Rate limiter triggers at limit",
      "passed": true,
      "latency_ms": 0,
      "notes": ""
    }
  ]
}