Hugging Face's logo

czty
/
Beyond_Prompt-based_Retrieval

Model card Files
xet
 Community
Beyond_Prompt-based_Retrieval
File size: 2,986 Bytes
c6856c9
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
1
2
3
4
5
6
7
8
9
10
11
12
13
14
15
16
17
18
19
20
21
22
23
24
25
26
27
28
29
30
31
32
33
34
35
36
37
38
39
40
41
42
43
44
45
46
47
48
49
50
51
52
53
54
55
56
57
58
59
60
61
62
63
64
65
66
67
68
69
70
71
72
73
74
75
76
77
78
79
80
81
82
83
84
85
86
87
88
89
90
91
92
93
94
95
96
97
98
99
100
101
102
103
104
105
106
107
108
109
{
  "mcp_root": "/225040511/project/Beyond Prompt-Based Planning/Biomanus/biomni_web/backend/data/mcp_generated",
  "catalog": "/225040511/project/Beyond Prompt-Based Planning/Biomanus/graph_outputs/mcp_generated_graph_all_20260522_124110/server_catalog.json",
  "server_count": 912,
  "catalog_tool_count": 3472,
  "sampled_tool_count": 12,
  "summary_table": [
    {
      "audit_layer": "Generated server entrypoint present",
      "unit": "mcp_generated server directories",
      "n_tested": 912,
      "n_success": 816,
      "success_rate": "89.5%",
      "notes": "server directory contains *_shim_server.py or *_server.py under app/"
    },
    {
      "audit_layer": "Server syntax/compile",
      "unit": "MCP servers",
      "n_tested": 816,
      "n_success": 816,
      "success_rate": "100.0%",
      "notes": "python -m py_compile on generated shim/raw server"
    },
    {
      "audit_layer": "Server import",
      "unit": "MCP servers",
      "n_tested": 816,
      "n_success": 799,
      "success_rate": "97.9%",
      "notes": "import generated server module in isolated process"
    },
    {
      "audit_layer": "MCP startup",
      "unit": "MCP servers",
      "n_tested": 816,
      "n_success": 816,
      "success_rate": "100.0%",
      "notes": "stdio server starts and does not immediately traceback; waiting servers count as success"
    },
    {
      "audit_layer": "Help/version probe",
      "unit": "servers with inferred underlying CLI",
      "n_tested": 519,
      "n_success": 174,
      "success_rate": "33.5%",
      "notes": "run inferred command with --help/-h/--version when command literal is found"
    },
    {
      "audit_layer": "Tiny fixture execution",
      "unit": "stratified sampled tools",
      "n_tested": 12,
      "n_success": 5,
      "success_rate": "41.7%",
      "notes": "sampled from 3472 catalog tools; required args filled with tiny FASTA/FASTQ/CSV/VCF fixtures"
    }
  ],
  "error_taxonomy": [
    {
      "scope": "server",
      "error_type": "missing_executable_or_file",
      "count": 345
    },
    {
      "scope": "server",
      "error_type": "missing_server_entrypoint",
      "count": 96
    },
    {
      "scope": "server",
      "error_type": "schema_or_argument_error",
      "count": 9
    },
    {
      "scope": "server",
      "error_type": "timeout",
      "count": 3
    },
    {
      "scope": "server",
      "error_type": "runtime_exception",
      "count": 2
    },
    {
      "scope": "server",
      "error_type": "missing_python_dependency",
      "count": 2
    },
    {
      "scope": "server",
      "error_type": "syntax_error",
      "count": 1
    },
    {
      "scope": "sampled_tool",
      "error_type": "missing_executable_or_file",
      "count": 4
    },
    {
      "scope": "sampled_tool",
      "error_type": "runtime_exception",
      "count": 2
    },
    {
      "scope": "sampled_tool",
      "error_type": "missing_python_dependency",
      "count": 1
    }
  ]
}