Biomanus/experiments/execution_audit/results_smoke2/summary.json

{
  "mcp_root": "/225040511/project/Beyond Prompt-Based Planning/Biomanus/biomni_web/backend/data/mcp_generated",
  "catalog": "/225040511/project/Beyond Prompt-Based Planning/Biomanus/graph_outputs/mcp_generated_graph_all_20260522_124110/server_catalog.json",
  "server_count": 912,
  "catalog_tool_count": 3472,
  "sampled_tool_count": 12,
  "summary_table": [
    {
      "audit_layer": "Generated server entrypoint present",
      "unit": "mcp_generated server directories",
      "n_tested": 912,
      "n_success": 816,
      "success_rate": "89.5%",
      "notes": "server directory contains *_shim_server.py or *_server.py under app/"
    },
    {
      "audit_layer": "Server syntax/compile",
      "unit": "MCP servers",
      "n_tested": 816,
      "n_success": 816,
      "success_rate": "100.0%",
      "notes": "python -m py_compile on generated shim/raw server"
    },
    {
      "audit_layer": "Server import",
      "unit": "MCP servers",
      "n_tested": 816,
      "n_success": 799,
      "success_rate": "97.9%",
      "notes": "import generated server module in isolated process"
    },
    {
      "audit_layer": "MCP startup",
      "unit": "MCP servers",
      "n_tested": 816,
      "n_success": 816,
      "success_rate": "100.0%",
      "notes": "stdio server starts and does not immediately traceback; waiting servers count as success"
    },
    {
      "audit_layer": "Help/version probe",
      "unit": "servers with inferred underlying CLI",
      "n_tested": 519,
      "n_success": 174,
      "success_rate": "33.5%",
      "notes": "run inferred command with --help/-h/--version when command literal is found"
    },
    {
      "audit_layer": "Tiny fixture execution",
      "unit": "stratified sampled tools",
      "n_tested": 12,
      "n_success": 5,
      "success_rate": "41.7%",
      "notes": "sampled from 3472 catalog tools; required args filled with tiny FASTA/FASTQ/CSV/VCF fixtures"
    }
  ],
  "error_taxonomy": [
    {
      "scope": "server",
      "error_type": "missing_executable_or_file",
      "count": 345
    },
    {
      "scope": "server",
      "error_type": "missing_server_entrypoint",
      "count": 96
    },
    {
      "scope": "server",
      "error_type": "schema_or_argument_error",
      "count": 9
    },
    {
      "scope": "server",
      "error_type": "timeout",
      "count": 3
    },
    {
      "scope": "server",
      "error_type": "runtime_exception",
      "count": 2
    },
    {
      "scope": "server",
      "error_type": "missing_python_dependency",
      "count": 2
    },
    {
      "scope": "server",
      "error_type": "syntax_error",
      "count": 1
    },
    {
      "scope": "sampled_tool",
      "error_type": "missing_executable_or_file",
      "count": 4
    },
    {
      "scope": "sampled_tool",
      "error_type": "runtime_exception",
      "count": 2
    },
    {
      "scope": "sampled_tool",
      "error_type": "missing_python_dependency",
      "count": 1
    }
  ]
}