| { |
| "mcp_root": "/225040511/project/Beyond Prompt-Based Planning/Biomanus/biomni_web/backend/data/mcp_generated", |
| "catalog": "/225040511/project/Beyond Prompt-Based Planning/Biomanus/graph_outputs/mcp_generated_graph_all_20260522_124110/server_catalog.json", |
| "server_count": 912, |
| "catalog_tool_count": 3472, |
| "sampled_tool_count": 300, |
| "summary_table": [ |
| { |
| "audit_layer": "Generated server entrypoint present", |
| "unit": "mcp_generated server directories", |
| "n_tested": 912, |
| "n_success": 816, |
| "success_rate": "89.5%", |
| "notes": "server directory contains *_shim_server.py or *_server.py under app/" |
| }, |
| { |
| "audit_layer": "Server syntax/compile", |
| "unit": "MCP servers", |
| "n_tested": 816, |
| "n_success": 816, |
| "success_rate": "100.0%", |
| "notes": "python -m py_compile on generated shim/raw server" |
| }, |
| { |
| "audit_layer": "Server import", |
| "unit": "MCP servers", |
| "n_tested": 816, |
| "n_success": 799, |
| "success_rate": "97.9%", |
| "notes": "import generated server module in isolated process" |
| }, |
| { |
| "audit_layer": "MCP startup", |
| "unit": "MCP servers", |
| "n_tested": 816, |
| "n_success": 816, |
| "success_rate": "100.0%", |
| "notes": "stdio server starts and does not immediately traceback; waiting servers count as success" |
| }, |
| { |
| "audit_layer": "Help/version probe", |
| "unit": "servers with inferred underlying CLI", |
| "n_tested": 519, |
| "n_success": 174, |
| "success_rate": "33.5%", |
| "notes": "run inferred command with --help/-h/--version when command literal is found" |
| }, |
| { |
| "audit_layer": "Tiny fixture execution", |
| "unit": "stratified sampled tools", |
| "n_tested": 300, |
| "n_success": 124, |
| "success_rate": "41.3%", |
| "notes": "sampled from 3472 catalog tools; required args filled with tiny FASTA/FASTQ/CSV/VCF fixtures" |
| } |
| ], |
| "error_taxonomy": [ |
| { |
| "scope": "server", |
| "error_type": "missing_executable_or_file", |
| "count": 345 |
| }, |
| { |
| "scope": "server", |
| "error_type": "missing_server_entrypoint", |
| "count": 96 |
| }, |
| { |
| "scope": "server", |
| "error_type": "schema_or_argument_error", |
| "count": 9 |
| }, |
| { |
| "scope": "server", |
| "error_type": "timeout", |
| "count": 3 |
| }, |
| { |
| "scope": "server", |
| "error_type": "runtime_exception", |
| "count": 2 |
| }, |
| { |
| "scope": "server", |
| "error_type": "missing_python_dependency", |
| "count": 2 |
| }, |
| { |
| "scope": "server", |
| "error_type": "syntax_error", |
| "count": 1 |
| }, |
| { |
| "scope": "sampled_tool", |
| "error_type": "missing_executable_or_file", |
| "count": 124 |
| }, |
| { |
| "scope": "sampled_tool", |
| "error_type": "runtime_exception", |
| "count": 44 |
| }, |
| { |
| "scope": "sampled_tool", |
| "error_type": "missing_python_dependency", |
| "count": 4 |
| }, |
| { |
| "scope": "sampled_tool", |
| "error_type": "timeout", |
| "count": 2 |
| }, |
| { |
| "scope": "sampled_tool", |
| "error_type": "runtime_nonzero_exit", |
| "count": 1 |
| }, |
| { |
| "scope": "sampled_tool", |
| "error_type": "schema_or_argument_error", |
| "count": 1 |
| } |
| ] |
| } |
