Hugging Face's logo

haochengsama
/
mcpuniverse

Model card Files
xet
 Community
haochengsama commited on
Commit
86acc45
·
verified ·
1 Parent(s): 43f5218

Add files using upload-large-folder tool

Browse files
Files changed (50) hide show
  1. .dockerignore +9 -0
  2. .env.example +89 -0
  3. .gitignore +279 -0
  4. .gitmodules +3 -0
  5. .playwright-mcp/console-2026-06-07T11-15-31-418Z.log +2 -0
  6. .playwright-mcp/page-2026-06-07T12-35-14-820Z.yml +1655 -0
  7. .pre-commit-config.yaml +9 -0
  8. .pylintrc +21 -0
  9. AI_ETHICS.md +13 -0
  10. CODEOWNERS +3 -0
  11. CODE_OF_CONDUCT.md +105 -0
  12. CONTRIBUTING.md +87 -0
  13. LICENSE.txt +207 -0
  14. Makefile +55 -0
  15. README.md +620 -1
  16. SECURITY.md +7 -0
  17. blender_addon.py +1458 -0
  18. console-errors-successful-load.log +4 -0
  19. dev-requirements.txt +5 -0
  20. docs/adding-mcp-servers.md +473 -0
  21. docs/blender-setup.md +135 -0
  22. docs/configuration-guide.md +541 -0
  23. docs/custom-agent-guide.md +297 -0
  24. docs/custom-evaluators-guide.md +260 -0
  25. docs/python-sandbox-setup.md +33 -0
  26. docs/system-architecture.md +275 -0
  27. docs/technical-blog.md +298 -0
  28. license_info.md +251 -0
  29. mcpuniverse.egg-info/PKG-INFO +731 -0
  30. mcpuniverse.egg-info/SOURCES.txt +437 -0
  31. mcpuniverse.egg-info/dependency_links.txt +1 -0
  32. mcpuniverse.egg-info/entry_points.txt +2 -0
  33. mcpuniverse.egg-info/top_level.txt +1 -0
  34. mcpuniverse/__init__.py +0 -0
  35. pyproject.toml +157 -0
  36. pytest.ini +4 -0
  37. requirements.txt +69 -0
  38. run_smoke_blender.py +52 -0
  39. run_smoke_browser.py +48 -0
  40. run_smoke_financial.py +45 -0
  41. setup_blender_and_vnc.sh +422 -0
  42. slime_mcp_rollout/.env.example +100 -0
  43. slime_mcp_rollout/README.md +364 -0
  44. slime_mcp_rollout/__init__.py +5 -0
  45. slime_mcp_rollout/llm_bridge.py +177 -0
  46. slime_mcp_rollout/rollout.py +107 -0
  47. slime_mcp_rollout/run_rollout_sft.py +78 -0
  48. slime_mcp_rollout/synthesis.py +418 -0
  49. slime_mcp_rollout/to_sft.py +138 -0
  50. sqlc.yaml +17 -0
.dockerignore ADDED
@@ -0,0 +1,9 @@
 
 
 
 
 
 
 
 
 
 
1
+ .git
2
+ __pycache__
3
+ *.pyc
4
+ *.egg-info
5
+ .pytest_cache
6
+ wandb
7
+ logs
8
+ *.log
9
+ mcpuniverse/experiments/
.env.example ADDED
@@ -0,0 +1,89 @@
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
1
+ # MCP gateway service address
2
+ MCP_GATEWAY_ADDRESS=http://localhost:8000
3
+ # Redis for Celery
4
+ REDIS_HOST=localhost
5
+ REDIS_PORT=6379
6
+ # Kafka MQ
7
+ KAFKA_HOST=localhost
8
+ KAFKA_PORT=9092
9
+ KAFKA_TOPIC=agent-task-mq
10
+
11
+ # SQLite tracer collector address
12
+ SQLITE_TRACER_COLLECTOR_ADDRESS=
13
+ SQLITE_CALLBACK_HANDLER_ADDRESS=
14
+
15
+ # LLM setup
16
+ OPENAI_API_KEY=
17
+ MISTRAL_API_KEY=
18
+ ANTHROPIC_API_KEY=
19
+ DEEPSEEK_API_KEY=
20
+ OLLAMA_URL=http://localhost:11434
21
+ SALESFORCE_GATEWAY_KEY=
22
+ GEMINI_API_KEY=
23
+ XAI_API_KEY=
24
+ OPENROUTER_API_KEY=
25
+ VLLM_API_KEY="token-abc123"
26
+ VLLM_BASE_URL="http://localhost:2024"
27
+
28
+ # Google search
29
+ SERP_API_KEY=
30
+ # Google search (Serper API)
31
+ SERPER_API_KEY=
32
+ SERPER_BASE_URL=
33
+ # Google maps
34
+ GOOGLE_MAPS_API_KEY=
35
+ # Github
36
+ GITHUB_PERSONAL_ACCESS_TOKEN=
37
+ GITHUB_PERSONAL_ACCOUNT_NAME=
38
+ # Google Sheets
39
+ GOOGLE_SERVICE_ACCOUNT_PATH=
40
+ GOOGLE_DRIVE_FOLDER_ID=
41
+ # Filesystem
42
+ FILESYSTEM_DIRECTORY=
43
+ # Slack
44
+ SLACK_BOT_TOKEN=
45
+ SLACK_TEAM_ID=
46
+ # Postgres
47
+ POSTGRES_ADDRESS=
48
+ # Notion
49
+ NOTION_API_KEY=
50
+ NOTION_ROOT_PAGE=
51
+ # JINA scraping with LLM summarization
52
+ JINA_API_KEY=
53
+ SUMMARY_LLM_BASE_URL=
54
+ SUMMARY_LLM_MODEL_NAME=
55
+ SUMMARY_LLM_API_KEY=
56
+ # Python sandbox
57
+ SANDBOX_HOST_PORT=
58
+ SANDBOX_ADDRESS=
59
+
60
+ # Blender
61
+ BLENDER_APP_PATH=
62
+ MCPUniverse_DIR=
63
+
64
+ # mcpmark configurations
65
+
66
+ ## mcpmark postgres
67
+ POSTGRES_HOST=localhost
68
+ POSTGRES_PORT=5432
69
+ POSTGRES_USERNAME=postgres
70
+ POSTGRES_PASSWORD=password
71
+
72
+ # mcpmark_filesystem
73
+ ## Filesystem MCP Server Configuration
74
+ # Both variables point to the same test environment directory
75
+ FILESYSTEM_TEST_DIR=./temp_environments
76
+ FILESYSTEM_TEST_ROOT=./test_environments
77
+
78
+ # mcpmark github
79
+ GITHUB_TOKENS=
80
+ MCP_GITHUB_TOKEN=
81
+ GITHUB_EVAL_ORG=
82
+
83
+ ## Notion
84
+ SOURCE_NOTION_API_KEY= # For Source Hub (templates)
85
+ EVAL_NOTION_API_KEY= # For Eval Hub (active evaluation)
86
+ SOURCE_PARENT_PAGE_TITLE= # Source hub page name (exact match)
87
+ EVAL_PARENT_PAGE_TITLE= # Must match the name of the empty page you created in Eval Hub
88
+ PLAYWRIGHT_BROWSER="chromium" # default to chromium, you can also choose firefox
89
+ PLAYWRIGHT_HEADLESS="True"
.gitignore ADDED
@@ -0,0 +1,279 @@
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
1
+ # Byte-compiled / optimized / DLL files
2
+ __pycache__/
3
+ *.py[cod]
4
+ *$py.class
5
+
6
+ # transient smoke-test artifacts
7
+ mcpuniverse/benchmark/configs/mcpuniverse/*_smoke.yaml
8
+ report_*.md
9
+
10
+ # C extensions
11
+ *.so
12
+
13
+ # Distribution / packaging
14
+ .Python
15
+ build/
16
+ develop-eggs/
17
+ dist/
18
+ downloads/
19
+ eggs/
20
+ .eggs/
21
+ lib/
22
+ lib64/
23
+ parts/
24
+ sdist/
25
+ var/
26
+ wheels/
27
+ share/python-wheels/
28
+ *.egg-info/
29
+ .installed.cfg
30
+ *.egg
31
+ MANIFEST
32
+
33
+ # PyInstaller
34
+ # Usually these files are written by a python script from a template
35
+ # before PyInstaller builds the exe, so as to inject date/other infos into it.
36
+ *.manifest
37
+ *.spec
38
+
39
+ # Installer logs
40
+ pip-log.txt
41
+ pip-delete-this-directory.txt
42
+
43
+ # Unit test / coverage reports
44
+ htmlcov/
45
+ .tox/
46
+ .nox/
47
+ .coverage
48
+ .coverage.*
49
+ .cache
50
+ nosetests.xml
51
+ coverage.xml
52
+ *.cover
53
+ *.py,cover
54
+ .hypothesis/
55
+ .pytest_cache/
56
+ cover/
57
+
58
+ # Claude code settings
59
+ .claude
60
+
61
+ # Translations
62
+ *.mo
63
+ *.pot
64
+
65
+ # Django stuff:
66
+ *.log
67
+ local_settings.py
68
+ db.sqlite3
69
+ db.sqlite3-journal
70
+
71
+ # Flask stuff:
72
+ instance/
73
+ .webassets-cache
74
+
75
+ # Scrapy stuff:
76
+ .scrapy
77
+
78
+ # Sphinx documentation
79
+ docs/_build/
80
+
81
+ # PyBuilder
82
+ .pybuilder/
83
+ target/
84
+
85
+ # Jupyter Notebook
86
+ .ipynb_checkpoints
87
+
88
+ # IPython
89
+ profile_default/
90
+ ipython_config.py
91
+
92
+ # pyenv
93
+ # For a library or package, you might want to ignore these files since the code is
94
+ # intended to run in multiple environments; otherwise, check them in:
95
+ # .python-version
96
+
97
+ # pipenv
98
+ # According to pypa/pipenv#598, it is recommended to include Pipfile.lock in version control.
99
+ # However, in case of collaboration, if having platform-specific dependencies or dependencies
100
+ # having no cross-platform support, pipenv may install dependencies that don't work, or not
101
+ # install all needed dependencies.
102
+ #Pipfile.lock
103
+
104
+ # UV
105
+ # Similar to Pipfile.lock, it is generally recommended to include uv.lock in version control.
106
+ # This is especially recommended for binary packages to ensure reproducibility, and is more
107
+ # commonly ignored for libraries.
108
+ #uv.lock
109
+
110
+ # poetry
111
+ # Similar to Pipfile.lock, it is generally recommended to include poetry.lock in version control.
112
+ # This is especially recommended for binary packages to ensure reproducibility, and is more
113
+ # commonly ignored for libraries.
114
+ # https://python-poetry.org/docs/basic-usage/#commit-your-poetrylock-file-to-version-control
115
+ #poetry.lock
116
+
117
+ # pdm
118
+ # Similar to Pipfile.lock, it is generally recommended to include pdm.lock in version control.
119
+ #pdm.lock
120
+ # pdm stores project-wide configurations in .pdm.toml, but it is recommended to not include it
121
+ # in version control.
122
+ # https://pdm.fming.dev/latest/usage/project/#working-with-version-control
123
+ .pdm.toml
124
+ .pdm-python
125
+ .pdm-build/
126
+
127
+ # PEP 582; used by e.g. github.com/David-OConnor/pyflow and github.com/pdm-project/pdm
128
+ __pypackages__/
129
+
130
+ # Celery stuff
131
+ celerybeat-schedule
132
+ celerybeat.pid
133
+
134
+ # SageMath parsed files
135
+ *.sage.py
136
+
137
+ # Environments
138
+ .env
139
+ .venv
140
+ env/
141
+ venv/
142
+ ENV/
143
+ env.bak/
144
+ venv.bak/
145
+
146
+ # Spyder project settings
147
+ .spyderproject
148
+ .spyproject
149
+
150
+ # Rope project settings
151
+ .ropeproject
152
+
153
+ # mkdocs documentation
154
+ /site
155
+
156
+ # mypy
157
+ .mypy_cache/
158
+ .dmypy.json
159
+ dmypy.json
160
+
161
+ # Pyre type checker
162
+ .pyre/
163
+
164
+ # pytype static type analyzer
165
+ .pytype/
166
+
167
+ # Cython debug symbols
168
+ cython_debug/
169
+
170
+ # PyCharm
171
+ # JetBrains specific template is maintained in a separate JetBrains.gitignore that can
172
+ # be found at https://github.com/github/gitignore/blob/main/Global/JetBrains.gitignore
173
+ # and can be added to the global gitignore or merged into this file. For a more nuclear
174
+ # option (not recommended) you can uncomment the following to ignore the entire idea folder.
175
+ #.idea/
176
+
177
+ # Ruff stuff:
178
+ .ruff_cache/
179
+
180
+ # PyPI configuration file
181
+ .pypirc
182
+
183
+
184
+ # Debug code
185
+ notebook/
186
+
187
+ # macos system file
188
+ **/.DS_Store
189
+
190
+ # debug logs
191
+ log/
192
+
193
+ # run script
194
+ run*.sh
195
+
196
+ # cursor results
197
+ cursor_results/
198
+
199
+ # other agent and llms
200
+ gpt_o3_results/
201
+ gpt_oss_results/
202
+ mcpuniverse/llm/gpt_oss_llm.py
203
+ tests/gpt-oss/
204
+ mcpuniverse/evaluator/blender/check_functions/evaluated_results/
205
+ mcpuniverse/evaluator/blender/blend_files/
206
+ tests/cursor/
207
+
208
+ # mcpmark test environments and data
209
+ test_environments/
210
+ temp_environments/
211
+ github_state/
212
+ tests/data/postgres/
213
+ tests/data/playwright_webarena/
214
+ tests/benchmark/*_run_many_llms*.py
215
+ mcpuniverse/benchmark/configs/deepresearch/browsecomp
216
+ mcpuniverse/benchmark/configs/deepresearch/gaia_val_text_only
217
+ mcpuniverse/benchmark/configs/deepresearch/hle_text_only
218
+
219
+ # verl training artifacts
220
+ mcpuniverse/rl/integrations/verl/outputs/
221
+ mcpuniverse/rl/integrations/verl/logs/
222
+ mcpuniverse/rl/integrations/verl/wandb/
223
+ mcpuniverse/rl/integrations/verl/checkpoints/
224
+ .nfs*
225
+
226
+ # dev tmp files
227
+ tests/mcp/logs
228
+ tests/mcp/test_gateway_stress.sh
229
+ tests/mcp/test_gateway_stress.py
230
+ tests/rl/integrations
231
+ mcpuniverse/experiments
232
+ todo
233
+ mcpuniverse/mcp/servers/yahoo_finance_new
234
+ mcpuniverse/benchmark/scripts
235
+ mcpuniverse/rl/examples/sglang_harmony_react.ipynb
236
+ mcpuniverse/rl/examples/sglang_tito_react_train.ipynb
237
+ mcpuniverse/rl/examples/sglang_react_train.ipynb
238
+ mcpuniverse/rl/data/yf_*.json
239
+ mcpuniverse/rl/integrations/verl/scripts
240
+ mcpuniverse/rl/integrations/verl/config/mcp_fully_async.yaml
241
+ mcpuniverse/rl/integrations/verl/config/mcp_gptoss_harmony_tito.yaml
242
+ # Blender install dir (downloaded by run_smoke_blender)
243
+ applications/
244
+
245
+ # Smoke test artifacts
246
+ mcpuniverse/benchmark/configs/mcpuniverse/*_smoke.yaml
247
+ report_*.md
248
+ log/
249
+ logs/
250
+ mcpuniverse/evaluator/blender/blend_files/
251
+ mcpuniverse/evaluator/blender/evaluated_results/
252
+
253
+ # Generated rollout trajectories
254
+ slime_mcp_rollout/data/trajectories/
255
+
256
+ # Per-domain task JSONL is host-specific: each line embeds an absolute
257
+ # `metadata.json_path` under mcpuniverse/benchmark/configs/... — so a JSONL
258
+ # committed on one host points at /home/<user>/... that doesn't exist on
259
+ # another clone, causing FileNotFoundError at rollout time. Regenerable
260
+ # any time via:
261
+ # python3 -m slime_mcp_rollout.data.prepare_data --domain <DOMAIN>
262
+ slime_mcp_rollout/data/*_sample.jsonl
263
+
264
+ # Augment-generated task JSONs are per-batch helper-LLM output, not
265
+ # reproducible across runs (non-zero temperature). Keep locally for
266
+ # inspection but don't track. The augment_tasks/ dir itself is preserved
267
+ # via .gitkeep so the path exists in fresh clones.
268
+ slime_mcp_rollout/augment_tasks/*/
269
+
270
+ # Artifacts left by @playwright/mcp at the repo root during browser_automation
271
+ .playwright-mcp/
272
+ # Screenshots / route maps written by playwright tasks at repo root.
273
+ # Any *.png at the repo root is a browser_automation artifact; legit
274
+ # logo/diagram PNGs live under assets/ or docs/_static/ and are not
275
+ # matched by this anchored pattern.
276
+ /*.png
277
+ route*.png
278
+ *.playwright-screenshot.png
279
+ slime_mcp_rollout/data/yf_cache/
.gitmodules ADDED
@@ -0,0 +1,3 @@
 
 
 
 
1
+ [submodule "third_party/mcpmark"]
2
+ path = third_party/mcpmark
3
+ url = git@github.com:eval-sys/mcpmark.git
.playwright-mcp/console-2026-06-07T11-15-31-418Z.log ADDED
@@ -0,0 +1,2 @@
 
 
 
1
+ [ 944ms] [ERROR] Failed to load resource: the server responded with a status of 404 () @ https://raw.githubusercontent.com/huggingface/datasets/main/datasets/gaia/gaia.py:0
2
+ [ 24704ms] [ERROR] Blocked script execution in 'https://raw.githubusercontent.com/huggingface/datasets/main/datasets/gaia/gaia.py' because the document's frame is sandboxed and the 'allow-scripts' permission is not set. @ https://raw.githubusercontent.com/huggingface/datasets/main/datasets/gaia/gaia.py:0
.playwright-mcp/page-2026-06-07T12-35-14-820Z.yml ADDED
@@ -0,0 +1,1655 @@
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
1
+ - generic [active] [ref=e1]:
2
+ - generic [ref=e2]:
3
+ - generic [ref=e3]:
4
+ - link "Skip to content" [ref=e4] [cursor=pointer]:
5
+ - /url: "#start-of-content"
6
+ - banner [ref=e6]:
7
+ - heading "Navigation Menu" [level=2] [ref=e7]
8
+ - generic [ref=e8]:
9
+ - link "Homepage" [ref=e10] [cursor=pointer]:
10
+ - /url: /
11
+ - img [ref=e11]
12
+ - generic [ref=e13]:
13
+ - navigation "Global" [ref=e16]:
14
+ - list [ref=e17]:
15
+ - listitem [ref=e18]:
16
+ - button "Platform" [ref=e20] [cursor=pointer]:
17
+ - text: Platform
18
+ - img [ref=e21]
19
+ - listitem [ref=e23]:
20
+ - button "Solutions" [ref=e25] [cursor=pointer]:
21
+ - text: Solutions
22
+ - img [ref=e26]
23
+ - listitem [ref=e28]:
24
+ - button "Resources" [ref=e30] [cursor=pointer]:
25
+ - text: Resources
26
+ - img [ref=e31]
27
+ - listitem [ref=e33]:
28
+ - button "Open Source" [ref=e35] [cursor=pointer]:
29
+ - text: Open Source
30
+ - img [ref=e36]
31
+ - listitem [ref=e38]:
32
+ - button "Enterprise" [ref=e40] [cursor=pointer]:
33
+ - text: Enterprise
34
+ - img [ref=e41]
35
+ - listitem [ref=e43]:
36
+ - link "Pricing" [ref=e44] [cursor=pointer]:
37
+ - /url: https://github.com/pricing
38
+ - generic [ref=e45]: Pricing
39
+ - generic [ref=e46]:
40
+ - button "Search or jump to…" [ref=e49] [cursor=pointer]:
41
+ - img [ref=e51]
42
+ - link "Sign in" [ref=e54] [cursor=pointer]:
43
+ - /url: /login?return_to=https%3A%2F%2Fgithub.com%2Fhuggingface%2Fagents-course%2Fblob%2Fmain%2Funits%2Fen%2F_toctree.yml
44
+ - link "Sign up" [ref=e55] [cursor=pointer]:
45
+ - /url: /signup?ref_cta=Sign+up&ref_loc=header+logged+out&ref_page=%2F%3Cuser-name%3E%2F%3Crepo-name%3E%2Fblob%2Fshow&source=header-repo&source_repo=huggingface%2Fagents-course
46
+ - button "Appearance settings" [ref=e58] [cursor=pointer]:
47
+ - img
48
+ - main [ref=e62]:
49
+ - generic [ref=e63]:
50
+ - generic [ref=e64]:
51
+ - generic [ref=e66]:
52
+ - img [ref=e67]
53
+ - link "huggingface" [ref=e70] [cursor=pointer]:
54
+ - /url: /huggingface
55
+ - generic [ref=e71]: /
56
+ - strong [ref=e72]:
57
+ - link "agents-course" [ref=e73] [cursor=pointer]:
58
+ - /url: /huggingface/agents-course
59
+ - generic [ref=e74]: Public
60
+ - generic [ref=e75]:
61
+ - list:
62
+ - listitem [ref=e76]:
63
+ - link "You must be signed in to change notification settings" [ref=e77] [cursor=pointer]:
64
+ - /url: /login?return_to=%2Fhuggingface%2Fagents-course
65
+ - img [ref=e78]
66
+ - text: Notifications
67
+ - listitem [ref=e80]:
68
+ - link "Fork 2.1k" [ref=e81] [cursor=pointer]:
69
+ - /url: /login?return_to=%2Fhuggingface%2Fagents-course
70
+ - img [ref=e82]
71
+ - text: Fork
72
+ - generic "2,096" [ref=e84]: 2.1k
73
+ - listitem [ref=e85]:
74
+ - link "You must be signed in to star a repository" [ref=e87] [cursor=pointer]:
75
+ - /url: /login?return_to=%2Fhuggingface%2Fagents-course
76
+ - img [ref=e88]
77
+ - text: Star
78
+ - generic "29174 users starred this repository" [ref=e90]: 29.2k
79
+ - navigation "Repository" [ref=e91]:
80
+ - list [ref=e92]:
81
+ - listitem [ref=e93]:
82
+ - link "Code" [ref=e94] [cursor=pointer]:
83
+ - /url: /huggingface/agents-course
84
+ - img [ref=e95]
85
+ - generic [ref=e97]: Code
86
+ - listitem [ref=e98]:
87
+ - link "Issues 86" [ref=e99] [cursor=pointer]:
88
+ - /url: /huggingface/agents-course/issues
89
+ - img [ref=e100]
90
+ - generic [ref=e103]: Issues
91
+ - generic "86" [ref=e104]
92
+ - listitem [ref=e105]:
93
+ - link "Pull requests 44" [ref=e106] [cursor=pointer]:
94
+ - /url: /huggingface/agents-course/pulls
95
+ - img [ref=e107]
96
+ - generic [ref=e109]: Pull requests
97
+ - generic "44" [ref=e110]
98
+ - listitem [ref=e111]:
99
+ - link "Actions" [ref=e112] [cursor=pointer]:
100
+ - /url: /huggingface/agents-course/actions
101
+ - img [ref=e113]
102
+ - generic [ref=e115]: Actions
103
+ - listitem [ref=e116]:
104
+ - link "Projects" [ref=e117] [cursor=pointer]:
105
+ - /url: /huggingface/agents-course/projects
106
+ - img [ref=e118]
107
+ - generic [ref=e120]: Projects
108
+ - listitem [ref=e121]:
109
+ - link "Security and quality" [ref=e122] [cursor=pointer]:
110
+ - /url: /huggingface/agents-course/security
111
+ - img [ref=e123]
112
+ - generic [ref=e125]: Security and quality
113
+ - listitem [ref=e126]:
114
+ - link "Insights" [ref=e127] [cursor=pointer]:
115
+ - /url: /huggingface/agents-course/pulse
116
+ - img [ref=e128]
117
+ - generic [ref=e130]: Insights
118
+ - generic [ref=e138]:
119
+ - generic [ref=e140]:
120
+ - generic [ref=e143]:
121
+ - generic [ref=e144]:
122
+ - generic [ref=e145]:
123
+ - heading "Collapse file tree" [level=2] [ref=e146]:
124
+ - button "Collapse file tree" [expanded] [ref=e147] [cursor=pointer]:
125
+ - img [ref=e148]
126
+ - heading "Files" [level=2] [ref=e151]
127
+ - generic [ref=e152]:
128
+ - button "main branch" [ref=e154] [cursor=pointer]:
129
+ - generic [ref=e155]:
130
+ - generic [ref=e157]:
131
+ - img [ref=e159]
132
+ - generic [ref=e162]: main
133
+ - generic:
134
+ - img
135
+ - button "Search this repository (forward slash)" [ref=e164] [cursor=pointer]:
136
+ - img [ref=e165]
137
+ - generic [ref=e168]:
138
+ - img [ref=e170]
139
+ - combobox "Go to file" [ref=e172]
140
+ - navigation "File Tree Navigation" [ref=e178]:
141
+ - tree "Files" [ref=e180]:
142
+ - treeitem ".github" [level=1] [ref=e181]:
143
+ - generic [ref=e182] [cursor=pointer]:
144
+ - img [ref=e184]
145
+ - generic [ref=e186]:
146
+ - img [ref=e190]
147
+ - generic [ref=e192]: .github
148
+ - treeitem "quiz" [level=1] [ref=e193]:
149
+ - generic [ref=e194] [cursor=pointer]:
150
+ - img [ref=e196]
151
+ - generic [ref=e198]:
152
+ - img [ref=e202]
153
+ - generic [ref=e204]: quiz
154
+ - treeitem "scripts" [level=1] [ref=e205]:
155
+ - generic [ref=e206] [cursor=pointer]:
156
+ - img [ref=e208]
157
+ - generic [ref=e210]:
158
+ - img [ref=e214]
159
+ - generic [ref=e216]: scripts
160
+ - treeitem "translation_agreements" [level=1] [ref=e217]:
161
+ - generic [ref=e218] [cursor=pointer]:
162
+ - img [ref=e220]
163
+ - generic [ref=e222]:
164
+ - img [ref=e226]
165
+ - generic [ref=e228]: translation_agreements
166
+ - treeitem "units" [expanded] [level=1] [ref=e229]:
167
+ - generic [ref=e230] [cursor=pointer]:
168
+ - img [ref=e232]
169
+ - generic [ref=e234]:
170
+ - img [ref=e238]
171
+ - generic [ref=e240]: units
172
+ - group "units" [ref=e241]:
173
+ - treeitem "en" [expanded] [level=2] [ref=e242]:
174
+ - generic [ref=e243] [cursor=pointer]:
175
+ - img [ref=e248]
176
+ - generic [ref=e250]:
177
+ - img [ref=e254]
178
+ - generic [ref=e256]: en
179
+ - group "en" [ref=e257]:
180
+ - treeitem "bonus-unit1" [level=3] [ref=e258]:
181
+ - generic [ref=e259] [cursor=pointer]:
182
+ - img [ref=e265]
183
+ - generic [ref=e267]:
184
+ - img [ref=e271]
185
+ - generic [ref=e273]: bonus-unit1
186
+ - treeitem "bonus-unit2" [level=3] [ref=e274]:
187
+ - generic [ref=e275] [cursor=pointer]:
188
+ - img [ref=e281]
189
+ - generic [ref=e283]:
190
+ - img [ref=e287]
191
+ - generic [ref=e289]: bonus-unit2
192
+ - treeitem "bonus-unit3" [level=3] [ref=e290]:
193
+ - generic [ref=e291] [cursor=pointer]:
194
+ - img [ref=e297]
195
+ - generic [ref=e299]:
196
+ - img [ref=e303]
197
+ - generic [ref=e305]: bonus-unit3
198
+ - treeitem "communication" [level=3] [ref=e306]:
199
+ - generic [ref=e307] [cursor=pointer]:
200
+ - img [ref=e313]
201
+ - generic [ref=e315]:
202
+ - img [ref=e319]
203
+ - generic [ref=e321]: communication
204
+ - treeitem "unit0" [level=3] [ref=e322]:
205
+ - generic [ref=e323] [cursor=pointer]:
206
+ - img [ref=e329]
207
+ - generic [ref=e331]:
208
+ - img [ref=e335]
209
+ - generic [ref=e337]: unit0
210
+ - treeitem "unit1" [level=3] [ref=e338]:
211
+ - generic [ref=e339] [cursor=pointer]:
212
+ - img [ref=e345]
213
+ - generic [ref=e347]:
214
+ - img [ref=e351]
215
+ - generic [ref=e353]: unit1
216
+ - treeitem "unit2" [level=3] [ref=e354]:
217
+ - generic [ref=e355] [cursor=pointer]:
218
+ - img [ref=e361]
219
+ - generic [ref=e363]:
220
+ - img [ref=e367]
221
+ - generic [ref=e369]: unit2
222
+ - treeitem "unit3" [level=3] [ref=e370]:
223
+ - generic [ref=e371] [cursor=pointer]:
224
+ - img [ref=e377]
225
+ - generic [ref=e379]:
226
+ - img [ref=e383]
227
+ - generic [ref=e385]: unit3
228
+ - treeitem "unit4" [level=3] [ref=e386]:
229
+ - generic [ref=e387] [cursor=pointer]:
230
+ - img [ref=e393]
231
+ - generic [ref=e395]:
232
+ - img [ref=e399]
233
+ - generic [ref=e401]: unit4
234
+ - treeitem "_toctree.yml" [level=3] [ref=e402]:
235
+ - generic [ref=e408] [cursor=pointer]:
236
+ - img [ref=e411]
237
+ - generic [ref=e413]: _toctree.yml
238
+ - treeitem "es" [level=2] [ref=e414]:
239
+ - generic [ref=e415] [cursor=pointer]:
240
+ - img [ref=e420]
241
+ - generic [ref=e422]:
242
+ - img [ref=e426]
243
+ - generic [ref=e428]: es
244
+ - treeitem "fr" [level=2] [ref=e429]:
245
+ - generic [ref=e430] [cursor=pointer]:
246
+ - img [ref=e435]
247
+ - generic [ref=e437]:
248
+ - img [ref=e441]
249
+ - generic [ref=e443]: fr
250
+ - treeitem "ko" [level=2] [ref=e444]:
251
+ - generic [ref=e445] [cursor=pointer]:
252
+ - img [ref=e450]
253
+ - generic [ref=e452]:
254
+ - img [ref=e456]
255
+ - generic [ref=e458]: ko
256
+ - treeitem "ru-RU" [level=2] [ref=e459]:
257
+ - generic [ref=e460] [cursor=pointer]:
258
+ - img [ref=e465]
259
+ - generic [ref=e467]:
260
+ - img [ref=e471]
261
+ - generic [ref=e473]: ru-RU
262
+ - treeitem "vi" [level=2] [ref=e474]:
263
+ - generic [ref=e475] [cursor=pointer]:
264
+ - img [ref=e480]
265
+ - generic [ref=e482]:
266
+ - img [ref=e486]
267
+ - generic [ref=e488]: vi
268
+ - treeitem "zh-CN" [level=2] [ref=e489]:
269
+ - generic [ref=e490] [cursor=pointer]:
270
+ - img [ref=e495]
271
+ - generic [ref=e497]:
272
+ - img [ref=e501]
273
+ - generic [ref=e503]: zh-CN
274
+ - treeitem ".gitignore" [level=1] [ref=e504]:
275
+ - generic [ref=e506] [cursor=pointer]:
276
+ - img [ref=e509]
277
+ - generic [ref=e511]: .gitignore
278
+ - treeitem "LICENSE" [level=1] [ref=e512]:
279
+ - generic [ref=e514] [cursor=pointer]:
280
+ - img [ref=e517]
281
+ - generic [ref=e519]: LICENSE
282
+ - treeitem "README.md" [level=1] [ref=e520]:
283
+ - generic [ref=e522] [cursor=pointer]:
284
+ - img [ref=e525]
285
+ - generic [ref=e527]: README.md
286
+ - slider "Draggable pane splitter" [ref=e529]
287
+ - generic [ref=e532]:
288
+ - generic [ref=e536]:
289
+ - generic [ref=e538]:
290
+ - navigation "Breadcrumbs" [ref=e539]:
291
+ - heading "Breadcrumbs" [level=2] [ref=e540]
292
+ - list [ref=e541]:
293
+ - listitem [ref=e542]:
294
+ - link "agents-course" [ref=e543] [cursor=pointer]:
295
+ - /url: /huggingface/agents-course/tree/main
296
+ - listitem [ref=e544]:
297
+ - text: /
298
+ - link "units" [ref=e545] [cursor=pointer]:
299
+ - /url: /huggingface/agents-course/tree/main/units
300
+ - listitem [ref=e546]:
301
+ - text: /
302
+ - link "en" [ref=e547] [cursor=pointer]:
303
+ - /url: /huggingface/agents-course/tree/main/units/en
304
+ - generic [ref=e548]:
305
+ - text: /
306
+ - heading "_toctree.yml" [level=1] [ref=e549]
307
+ - button "Copy path" [ref=e550] [cursor=pointer]:
308
+ - img [ref=e551]
309
+ - button "More file actions" [ref=e557] [cursor=pointer]:
310
+ - img [ref=e558]
311
+ - generic [ref=e560]:
312
+ - generic [ref=e562]:
313
+ - heading "Latest commit" [level=2] [ref=e563]
314
+ - generic [ref=e564]:
315
+ - generic [ref=e566]:
316
+ - link "sergiopaniego" [ref=e567] [cursor=pointer]:
317
+ - /url: /sergiopaniego
318
+ - img "sergiopaniego" [ref=e568]
319
+ - link "commits by sergiopaniego" [ref=e569] [cursor=pointer]:
320
+ - /url: /huggingface/agents-course/commits?author=sergiopaniego
321
+ - text: sergiopaniego
322
+ - generic [ref=e570]:
323
+ - link "Merge branch 'main' into BonusUnit2/AIInGames" [ref=e573] [cursor=pointer]:
324
+ - /url: /huggingface/agents-course/commit/e9d6f6cecd0c1b07af9a5dd218a15d84368b21a4
325
+ - button "success" [ref=e574] [cursor=pointer]:
326
+ - img [ref=e575]
327
+ - generic [ref=e577]:
328
+ - generic [ref=e579]:
329
+ - link "Commit e9d6f6c" [ref=e580] [cursor=pointer]:
330
+ - /url: /huggingface/agents-course/commit/e9d6f6cecd0c1b07af9a5dd218a15d84368b21a4
331
+ - text: e9d6f6c
332
+ - text: ·
333
+ - generic "Apr 25, 2025, 12:23 PM UTC" [ref=e581]: Apr 25, 2025last year
334
+ - generic [ref=e582]:
335
+ - heading "History" [level=2] [ref=e583]
336
+ - link "History" [ref=e584] [cursor=pointer]:
337
+ - /url: /huggingface/agents-course/commits/main/units/en/_toctree.yml
338
+ - generic [ref=e585]:
339
+ - generic:
340
+ - img
341
+ - generic [ref=e586]: History
342
+ - generic [ref=e588]:
343
+ - generic [ref=e590]:
344
+ - heading "File metadata and controls" [level=2] [ref=e591]
345
+ - list "File view" [ref=e593]:
346
+ - listitem [ref=e594]:
347
+ - button "Code" [ref=e595] [cursor=pointer]:
348
+ - generic [ref=e597]: Code
349
+ - listitem [ref=e598]:
350
+ - button "Blame" [ref=e599] [cursor=pointer]:
351
+ - generic [ref=e601]: Blame
352
+ - generic [ref=e602]:
353
+ - generic [ref=e603]:
354
+ - generic [ref=e604]:
355
+ - link "Raw" [ref=e606] [cursor=pointer]:
356
+ - /url: https://github.com/huggingface/agents-course/raw/refs/heads/main/units/en/_toctree.yml
357
+ - generic [ref=e608]: Raw
358
+ - button "Copy raw file" [ref=e610] [cursor=pointer]:
359
+ - img [ref=e611]
360
+ - button "Download raw file" [ref=e615] [cursor=pointer]:
361
+ - img [ref=e616]
362
+ - generic [ref=e619]:
363
+ - button "You must be signed in to make or propose changes" [disabled] [ref=e621]:
364
+ - img [ref=e622]
365
+ - button "More edit options" [ref=e625] [cursor=pointer]:
366
+ - img [ref=e626]
367
+ - button "Open symbols panel" [ref=e628] [cursor=pointer]:
368
+ - img [ref=e629]
369
+ - region "_toctree.yml" [ref=e632]:
370
+ - generic [ref=e636]:
371
+ - textbox "file content" [ref=e641]: "- title: Unit 0. Welcome to the course sections: - local: unit0/introduction title: Welcome to the course 🤗 - local: unit0/onboarding title: Onboarding - local: unit0/discord101 title: (Optional) Discord 101 - title: Live 1. How the course works and Q&A sections: - local: communication/live1 title: Live 1. How the course works and Q&A - title: Unit 1. Introduction to Agents sections: - local: unit1/introduction title: Introduction - local: unit1/what-are-agents title: What is an Agent? - local: unit1/quiz1 title: Quick Quiz 1 - local: unit1/what-are-llms title: What are LLMs? - local: unit1/messages-and-special-tokens title: Messages and Special Tokens - local: unit1/tools title: What are Tools? - local: unit1/quiz2 title: Quick Quiz 2 - local: unit1/agent-steps-and-structure title: Understanding AI Agents through the Thought-Action-Observation Cycle - local: unit1/thoughts title: Thought, Internal Reasoning and the Re-Act Approach - local: unit1/actions title: Actions, Enabling the Agent to Engage with Its Environment - local: unit1/observations title: Observe, Integrating Feedback to Reflect and Adapt - local: unit1/dummy-agent-library title: Dummy Agent Library - local: unit1/tutorial title: Let’s Create Our First Agent Using smolagents - local: unit1/final-quiz title: Unit 1 Final Quiz - local: unit1/conclusion title: Conclusion - title: Unit 2. Frameworks for AI Agents sections: - local: unit2/introduction title: Frameworks for AI Agents - title: Unit 2.1 The smolagents framework sections: - local: unit2/smolagents/introduction title: Introduction to smolagents - local: unit2/smolagents/why_use_smolagents title: Why use smolagents? - local: unit2/smolagents/quiz1 title: Quick Quiz 1 - local: unit2/smolagents/code_agents title: Building Agents That Use Code - local: unit2/smolagents/tool_calling_agents title: Writing actions as code snippets or JSON blobs - local: unit2/smolagents/tools title: Tools - local: unit2/smolagents/retrieval_agents title: Retrieval Agents - local: unit2/smolagents/quiz2 title: Quick Quiz 2 - local: unit2/smolagents/multi_agent_systems title: Multi-Agent Systems - local: unit2/smolagents/vision_agents title: Vision and Browser agents - local: unit2/smolagents/final_quiz title: Final Quiz - local: unit2/smolagents/conclusion title: Conclusion - title: Unit 2.2 The LlamaIndex framework sections: - local: unit2/llama-index/introduction title: Introduction to LLamaIndex - local: unit2/llama-index/llama-hub title: Introduction to LlamaHub - local: unit2/llama-index/components title: What are Components in LlamaIndex? - local: unit2/llama-index/tools title: Using Tools in LlamaIndex - local: unit2/llama-index/quiz1 title: Quick Quiz 1 - local: unit2/llama-index/agents title: Using Agents in LlamaIndex - local: unit2/llama-index/workflows title: Creating Agentic Workflows in LlamaIndex - local: unit2/llama-index/quiz2 title: Quick Quiz 2 - local: unit2/llama-index/conclusion title: Conclusion - title: Unit 2.3 The LangGraph framework sections: - local: unit2/langgraph/introduction title: Introduction to LangGraph - local: unit2/langgraph/when_to_use_langgraph title: What is LangGraph? - local: unit2/langgraph/building_blocks title: Building Blocks of LangGraph - local: unit2/langgraph/first_graph title: Building Your First LangGraph - local: unit2/langgraph/document_analysis_agent title: Document Analysis Graph - local: unit2/langgraph/quiz1 title: Quick Quiz 1 - local: unit2/langgraph/conclusion title: Conclusion - title: Unit 3. Use Case for Agentic RAG sections: - local: unit3/agentic-rag/introduction title: Introduction to Use Case for Agentic RAG - local: unit3/agentic-rag/agentic-rag title: Agentic Retrieval Augmented Generation (RAG) - local: unit3/agentic-rag/invitees title: Creating a RAG Tool for Guest Stories - local: unit3/agentic-rag/tools title: Building and Integrating Tools for Your Agent - local: unit3/agentic-rag/agent title: Creating Your Gala Agent - local: unit3/agentic-rag/conclusion title: Conclusion - title: Unit 4. Final Project - Create, Test, and Certify Your Agent sections: - local: unit4/introduction title: Introduction to the Final Unit - local: unit4/what-is-gaia title: What is GAIA? - local: unit4/hands-on title: The Final Hands-On - local: unit4/get-your-certificate title: Get Your Certificate Of Excellence - local: unit4/conclusion title: Conclusion of the Course - local: unit4/additional-readings title: What Should You Learn Now? - title: Bonus Unit 1. Fine-tuning an LLM for Function-calling sections: - local: bonus-unit1/introduction title: Introduction - local: bonus-unit1/what-is-function-calling title: What is Function Calling? - local: bonus-unit1/fine-tuning title: Let's Fine-Tune your model for Function-calling - local: bonus-unit1/conclusion title: Conclusion - title: Bonus Unit 2. Agent Observability and Evaluation sections: - local: bonus-unit2/introduction title: Introduction - local: bonus-unit2/what-is-agent-observability-and-evaluation title: What is agent observability and evaluation? - local: bonus-unit2/monitoring-and-evaluating-agents-notebook title: Monitoring and evaluating agents - local: bonus-unit2/quiz title: Quiz - title: Bonus Unit 3. Agents in Games with Pokemon sections: - local: bonus-unit3/introduction title: Introduction - local: bonus-unit3/state-of-art title: The State of the Art in Using LLMs in Games - local: bonus-unit3/from-llm-to-agents title: From LLMs to AI Agents - local: bonus-unit3/building_your_pokemon_agent title: Build Your Own Pokémon Battle Agent - local: bonus-unit3/launching_agent_battle title: Launching Your Pokémon Battle Agent - local: bonus-unit3/conclusion title: Conclusion"
372
+ - generic:
373
+ - generic:
374
+ - generic:
375
+ - generic [ref=e642]:
376
+ - generic [ref=e643] [cursor=pointer]: "1"
377
+ - generic [ref=e644] [cursor=pointer]: "2"
378
+ - generic [ref=e645] [cursor=pointer]: "3"
379
+ - generic [ref=e646] [cursor=pointer]: "4"
380
+ - generic [ref=e647] [cursor=pointer]: "5"
381
+ - generic [ref=e648] [cursor=pointer]: "6"
382
+ - generic [ref=e649] [cursor=pointer]: "7"
383
+ - generic [ref=e650] [cursor=pointer]: "8"
384
+ - generic [ref=e651] [cursor=pointer]: "9"
385
+ - generic [ref=e652] [cursor=pointer]: "10"
386
+ - generic [ref=e653] [cursor=pointer]: "11"
387
+ - generic [ref=e654] [cursor=pointer]: "12"
388
+ - generic [ref=e655] [cursor=pointer]: "13"
389
+ - generic [ref=e656] [cursor=pointer]: "14"
390
+ - generic [ref=e657] [cursor=pointer]: "15"
391
+ - generic [ref=e658] [cursor=pointer]: "16"
392
+ - generic [ref=e659] [cursor=pointer]: "17"
393
+ - generic [ref=e660] [cursor=pointer]: "18"
394
+ - generic [ref=e661] [cursor=pointer]: "19"
395
+ - generic [ref=e662] [cursor=pointer]: "20"
396
+ - generic [ref=e663] [cursor=pointer]: "21"
397
+ - generic [ref=e664] [cursor=pointer]: "22"
398
+ - generic [ref=e665] [cursor=pointer]: "23"
399
+ - generic [ref=e666] [cursor=pointer]: "24"
400
+ - generic [ref=e667] [cursor=pointer]: "25"
401
+ - generic [ref=e668] [cursor=pointer]: "26"
402
+ - generic [ref=e669] [cursor=pointer]: "27"
403
+ - generic [ref=e670] [cursor=pointer]: "28"
404
+ - generic [ref=e671] [cursor=pointer]: "29"
405
+ - generic [ref=e672] [cursor=pointer]: "30"
406
+ - generic [ref=e673] [cursor=pointer]: "31"
407
+ - generic [ref=e674] [cursor=pointer]: "32"
408
+ - generic [ref=e675] [cursor=pointer]: "33"
409
+ - generic [ref=e676] [cursor=pointer]: "34"
410
+ - generic [ref=e677] [cursor=pointer]: "35"
411
+ - generic [ref=e678] [cursor=pointer]: "36"
412
+ - generic [ref=e679] [cursor=pointer]: "37"
413
+ - generic [ref=e680] [cursor=pointer]: "38"
414
+ - generic [ref=e681] [cursor=pointer]: "39"
415
+ - generic [ref=e682] [cursor=pointer]: "40"
416
+ - generic [ref=e683] [cursor=pointer]: "41"
417
+ - generic [ref=e684] [cursor=pointer]: "42"
418
+ - generic [ref=e685] [cursor=pointer]: "43"
419
+ - generic [ref=e686] [cursor=pointer]: "44"
420
+ - generic [ref=e687] [cursor=pointer]: "45"
421
+ - generic [ref=e688] [cursor=pointer]: "46"
422
+ - generic [ref=e689] [cursor=pointer]: "47"
423
+ - generic [ref=e690] [cursor=pointer]: "48"
424
+ - generic [ref=e691] [cursor=pointer]: "49"
425
+ - generic [ref=e692] [cursor=pointer]: "50"
426
+ - generic [ref=e693] [cursor=pointer]: "51"
427
+ - generic [ref=e694] [cursor=pointer]: "52"
428
+ - generic [ref=e695] [cursor=pointer]: "53"
429
+ - generic [ref=e696] [cursor=pointer]: "54"
430
+ - generic [ref=e697] [cursor=pointer]: "55"
431
+ - generic [ref=e698] [cursor=pointer]: "56"
432
+ - generic [ref=e699] [cursor=pointer]: "57"
433
+ - generic [ref=e700] [cursor=pointer]: "58"
434
+ - generic [ref=e701] [cursor=pointer]: "59"
435
+ - generic [ref=e702] [cursor=pointer]: "60"
436
+ - generic [ref=e703] [cursor=pointer]: "61"
437
+ - generic [ref=e704] [cursor=pointer]: "62"
438
+ - generic [ref=e705] [cursor=pointer]: "63"
439
+ - generic [ref=e706] [cursor=pointer]: "64"
440
+ - generic [ref=e707] [cursor=pointer]: "65"
441
+ - generic [ref=e708] [cursor=pointer]: "66"
442
+ - generic [ref=e709] [cursor=pointer]: "67"
443
+ - generic [ref=e710] [cursor=pointer]: "68"
444
+ - generic [ref=e711] [cursor=pointer]: "69"
445
+ - generic [ref=e712] [cursor=pointer]: "70"
446
+ - generic [ref=e713] [cursor=pointer]: "71"
447
+ - generic [ref=e714] [cursor=pointer]: "72"
448
+ - generic [ref=e715] [cursor=pointer]: "73"
449
+ - generic [ref=e716] [cursor=pointer]: "74"
450
+ - generic [ref=e717] [cursor=pointer]: "75"
451
+ - generic [ref=e718] [cursor=pointer]: "76"
452
+ - generic [ref=e719] [cursor=pointer]: "77"
453
+ - generic [ref=e720] [cursor=pointer]: "78"
454
+ - generic [ref=e721] [cursor=pointer]: "79"
455
+ - generic [ref=e722] [cursor=pointer]: "80"
456
+ - generic [ref=e723] [cursor=pointer]: "81"
457
+ - generic [ref=e724] [cursor=pointer]: "82"
458
+ - generic [ref=e725] [cursor=pointer]: "83"
459
+ - generic [ref=e726] [cursor=pointer]: "84"
460
+ - generic [ref=e727] [cursor=pointer]: "85"
461
+ - generic [ref=e728] [cursor=pointer]: "86"
462
+ - generic [ref=e729] [cursor=pointer]: "87"
463
+ - generic [ref=e730] [cursor=pointer]: "88"
464
+ - generic [ref=e731] [cursor=pointer]: "89"
465
+ - generic [ref=e732] [cursor=pointer]: "90"
466
+ - generic [ref=e733] [cursor=pointer]: "91"
467
+ - generic [ref=e734] [cursor=pointer]: "92"
468
+ - generic [ref=e735] [cursor=pointer]: "93"
469
+ - generic [ref=e736] [cursor=pointer]: "94"
470
+ - generic [ref=e737] [cursor=pointer]: "95"
471
+ - generic [ref=e738] [cursor=pointer]: "96"
472
+ - generic [ref=e739] [cursor=pointer]: "97"
473
+ - generic [ref=e740] [cursor=pointer]: "98"
474
+ - generic [ref=e741] [cursor=pointer]: "99"
475
+ - generic [ref=e742] [cursor=pointer]: "100"
476
+ - generic [ref=e743] [cursor=pointer]: "101"
477
+ - generic [ref=e744] [cursor=pointer]: "102"
478
+ - generic [ref=e745] [cursor=pointer]: "103"
479
+ - generic [ref=e746] [cursor=pointer]: "104"
480
+ - generic [ref=e747] [cursor=pointer]: "105"
481
+ - generic [ref=e748] [cursor=pointer]: "106"
482
+ - generic [ref=e749] [cursor=pointer]: "107"
483
+ - generic [ref=e750] [cursor=pointer]: "108"
484
+ - generic [ref=e751] [cursor=pointer]: "109"
485
+ - generic [ref=e752] [cursor=pointer]: "110"
486
+ - generic [ref=e753] [cursor=pointer]: "111"
487
+ - generic [ref=e754] [cursor=pointer]: "112"
488
+ - generic [ref=e755] [cursor=pointer]: "113"
489
+ - generic [ref=e756] [cursor=pointer]: "114"
490
+ - generic [ref=e757] [cursor=pointer]: "115"
491
+ - generic [ref=e758] [cursor=pointer]: "116"
492
+ - generic [ref=e759] [cursor=pointer]: "117"
493
+ - generic [ref=e760] [cursor=pointer]: "118"
494
+ - generic [ref=e761] [cursor=pointer]: "119"
495
+ - generic [ref=e762] [cursor=pointer]: "120"
496
+ - generic [ref=e763] [cursor=pointer]: "121"
497
+ - generic [ref=e764] [cursor=pointer]: "122"
498
+ - generic [ref=e765] [cursor=pointer]: "123"
499
+ - generic [ref=e766] [cursor=pointer]: "124"
500
+ - generic [ref=e767] [cursor=pointer]: "125"
501
+ - generic [ref=e768] [cursor=pointer]: "126"
502
+ - generic [ref=e769] [cursor=pointer]: "127"
503
+ - generic [ref=e770] [cursor=pointer]: "128"
504
+ - generic [ref=e771] [cursor=pointer]: "129"
505
+ - generic [ref=e772] [cursor=pointer]: "130"
506
+ - generic [ref=e773] [cursor=pointer]: "131"
507
+ - generic [ref=e774] [cursor=pointer]: "132"
508
+ - generic [ref=e775] [cursor=pointer]: "133"
509
+ - generic [ref=e776] [cursor=pointer]: "134"
510
+ - generic [ref=e777] [cursor=pointer]: "135"
511
+ - generic [ref=e778] [cursor=pointer]: "136"
512
+ - generic [ref=e779] [cursor=pointer]: "137"
513
+ - generic [ref=e780] [cursor=pointer]: "138"
514
+ - generic [ref=e781] [cursor=pointer]: "139"
515
+ - generic [ref=e782] [cursor=pointer]: "140"
516
+ - generic [ref=e783] [cursor=pointer]: "141"
517
+ - generic [ref=e784] [cursor=pointer]: "142"
518
+ - generic [ref=e785] [cursor=pointer]: "143"
519
+ - generic [ref=e786] [cursor=pointer]: "144"
520
+ - generic [ref=e787] [cursor=pointer]: "145"
521
+ - generic [ref=e788] [cursor=pointer]: "146"
522
+ - generic [ref=e789] [cursor=pointer]: "147"
523
+ - generic [ref=e790] [cursor=pointer]: "148"
524
+ - generic [ref=e791] [cursor=pointer]: "149"
525
+ - generic [ref=e792] [cursor=pointer]: "150"
526
+ - generic [ref=e793] [cursor=pointer]: "151"
527
+ - generic [ref=e794] [cursor=pointer]: "152"
528
+ - generic [ref=e795] [cursor=pointer]: "153"
529
+ - generic [ref=e796] [cursor=pointer]: "154"
530
+ - generic [ref=e797] [cursor=pointer]: "155"
531
+ - generic [ref=e798] [cursor=pointer]: "156"
532
+ - generic [ref=e799] [cursor=pointer]: "157"
533
+ - generic [ref=e800] [cursor=pointer]: "158"
534
+ - generic [ref=e801] [cursor=pointer]: "159"
535
+ - generic [ref=e802] [cursor=pointer]: "160"
536
+ - generic [ref=e803] [cursor=pointer]: "161"
537
+ - generic [ref=e804] [cursor=pointer]: "162"
538
+ - generic [ref=e805] [cursor=pointer]: "163"
539
+ - generic [ref=e806] [cursor=pointer]: "164"
540
+ - generic [ref=e807] [cursor=pointer]: "165"
541
+ - generic [ref=e808] [cursor=pointer]: "166"
542
+ - generic [ref=e809] [cursor=pointer]: "167"
543
+ - generic [ref=e810] [cursor=pointer]: "168"
544
+ - generic [ref=e811] [cursor=pointer]: "169"
545
+ - generic [ref=e812] [cursor=pointer]: "170"
546
+ - generic [ref=e813] [cursor=pointer]: "171"
547
+ - generic [ref=e814] [cursor=pointer]: "172"
548
+ - generic:
549
+ - generic:
550
+ - generic:
551
+ - generic:
552
+ - text: "-"
553
+ - generic: title
554
+ - text: ":"
555
+ - generic: Unit 0. Welcome to the course
556
+ - generic:
557
+ - generic:
558
+ - generic:
559
+ - generic: sections
560
+ - text: ":"
561
+ - generic:
562
+ - generic:
563
+ - generic:
564
+ - text: "-"
565
+ - generic: local
566
+ - text: ":"
567
+ - generic: unit0/introduction
568
+ - generic:
569
+ - generic:
570
+ - generic:
571
+ - generic: title
572
+ - text: ":"
573
+ - generic: Welcome to the course 🤗
574
+ - generic:
575
+ - generic:
576
+ - generic:
577
+ - text: "-"
578
+ - generic: local
579
+ - text: ":"
580
+ - generic: unit0/onboarding
581
+ - generic:
582
+ - generic:
583
+ - generic:
584
+ - generic: title
585
+ - text: ":"
586
+ - generic: Onboarding
587
+ - generic:
588
+ - generic:
589
+ - generic:
590
+ - text: "-"
591
+ - generic: local
592
+ - text: ":"
593
+ - generic: unit0/discord101
594
+ - generic:
595
+ - generic:
596
+ - generic:
597
+ - generic: title
598
+ - text: ":"
599
+ - generic: (Optional) Discord 101
600
+ - generic:
601
+ - generic:
602
+ - generic:
603
+ - text: "-"
604
+ - generic: title
605
+ - text: ":"
606
+ - generic: Live 1. How the course works and Q&A
607
+ - generic:
608
+ - generic:
609
+ - generic:
610
+ - generic: sections
611
+ - text: ":"
612
+ - generic:
613
+ - generic:
614
+ - generic:
615
+ - text: "-"
616
+ - generic: local
617
+ - text: ":"
618
+ - generic: communication/live1
619
+ - generic:
620
+ - generic:
621
+ - generic:
622
+ - generic: title
623
+ - text: ":"
624
+ - generic: Live 1. How the course works and Q&A
625
+ - generic:
626
+ - generic:
627
+ - generic:
628
+ - text: "-"
629
+ - generic: title
630
+ - text: ":"
631
+ - generic: Unit 1. Introduction to Agents
632
+ - generic:
633
+ - generic:
634
+ - generic:
635
+ - generic: sections
636
+ - text: ":"
637
+ - generic:
638
+ - generic:
639
+ - generic:
640
+ - text: "-"
641
+ - generic: local
642
+ - text: ":"
643
+ - generic: unit1/introduction
644
+ - generic:
645
+ - generic:
646
+ - generic:
647
+ - generic: title
648
+ - text: ":"
649
+ - generic: Introduction
650
+ - generic:
651
+ - generic:
652
+ - generic:
653
+ - text: "-"
654
+ - generic: local
655
+ - text: ":"
656
+ - generic: unit1/what-are-agents
657
+ - generic:
658
+ - generic:
659
+ - generic:
660
+ - generic: title
661
+ - text: ":"
662
+ - generic: What is an Agent?
663
+ - generic:
664
+ - generic:
665
+ - generic:
666
+ - text: "-"
667
+ - generic: local
668
+ - text: ":"
669
+ - generic: unit1/quiz1
670
+ - generic:
671
+ - generic:
672
+ - generic:
673
+ - generic: title
674
+ - text: ":"
675
+ - generic: Quick Quiz 1
676
+ - generic:
677
+ - generic:
678
+ - generic:
679
+ - text: "-"
680
+ - generic: local
681
+ - text: ":"
682
+ - generic: unit1/what-are-llms
683
+ - generic:
684
+ - generic:
685
+ - generic:
686
+ - generic: title
687
+ - text: ":"
688
+ - generic: What are LLMs?
689
+ - generic:
690
+ - generic:
691
+ - generic:
692
+ - text: "-"
693
+ - generic: local
694
+ - text: ":"
695
+ - generic: unit1/messages-and-special-tokens
696
+ - generic:
697
+ - generic:
698
+ - generic:
699
+ - generic: title
700
+ - text: ":"
701
+ - generic: Messages and Special Tokens
702
+ - generic:
703
+ - generic:
704
+ - generic:
705
+ - text: "-"
706
+ - generic: local
707
+ - text: ":"
708
+ - generic: unit1/tools
709
+ - generic:
710
+ - generic:
711
+ - generic:
712
+ - generic: title
713
+ - text: ":"
714
+ - generic: What are Tools?
715
+ - generic:
716
+ - generic:
717
+ - generic:
718
+ - text: "-"
719
+ - generic: local
720
+ - text: ":"
721
+ - generic: unit1/quiz2
722
+ - generic:
723
+ - generic:
724
+ - generic:
725
+ - generic: title
726
+ - text: ":"
727
+ - generic: Quick Quiz 2
728
+ - generic:
729
+ - generic:
730
+ - generic:
731
+ - text: "-"
732
+ - generic: local
733
+ - text: ":"
734
+ - generic: unit1/agent-steps-and-structure
735
+ - generic:
736
+ - generic:
737
+ - generic:
738
+ - generic: title
739
+ - text: ":"
740
+ - generic: Understanding AI Agents through the Thought-Action-Observation Cycle
741
+ - generic:
742
+ - generic:
743
+ - generic:
744
+ - text: "-"
745
+ - generic: local
746
+ - text: ":"
747
+ - generic: unit1/thoughts
748
+ - generic:
749
+ - generic:
750
+ - generic:
751
+ - generic: title
752
+ - text: ":"
753
+ - generic: Thought, Internal Reasoning and the Re-Act Approach
754
+ - generic:
755
+ - generic:
756
+ - generic:
757
+ - text: "-"
758
+ - generic: local
759
+ - text: ":"
760
+ - generic: unit1/actions
761
+ - generic:
762
+ - generic:
763
+ - generic:
764
+ - generic: title
765
+ - text: ":"
766
+ - generic: Actions, Enabling the Agent to Engage with Its Environment
767
+ - generic:
768
+ - generic:
769
+ - generic:
770
+ - text: "-"
771
+ - generic: local
772
+ - text: ":"
773
+ - generic: unit1/observations
774
+ - generic:
775
+ - generic:
776
+ - generic:
777
+ - generic: title
778
+ - text: ":"
779
+ - generic: Observe, Integrating Feedback to Reflect and Adapt
780
+ - generic:
781
+ - generic:
782
+ - generic:
783
+ - text: "-"
784
+ - generic: local
785
+ - text: ":"
786
+ - generic: unit1/dummy-agent-library
787
+ - generic:
788
+ - generic:
789
+ - generic:
790
+ - generic: title
791
+ - text: ":"
792
+ - generic: Dummy Agent Library
793
+ - generic:
794
+ - generic:
795
+ - generic:
796
+ - text: "-"
797
+ - generic: local
798
+ - text: ":"
799
+ - generic: unit1/tutorial
800
+ - generic:
801
+ - generic:
802
+ - generic:
803
+ - generic: title
804
+ - text: ":"
805
+ - generic: Let’s Create Our First Agent Using smolagents
806
+ - generic:
807
+ - generic:
808
+ - generic:
809
+ - text: "-"
810
+ - generic: local
811
+ - text: ":"
812
+ - generic: unit1/final-quiz
813
+ - generic:
814
+ - generic:
815
+ - generic:
816
+ - generic: title
817
+ - text: ":"
818
+ - generic: Unit 1 Final Quiz
819
+ - generic:
820
+ - generic:
821
+ - generic:
822
+ - text: "-"
823
+ - generic: local
824
+ - text: ":"
825
+ - generic: unit1/conclusion
826
+ - generic:
827
+ - generic:
828
+ - generic:
829
+ - generic: title
830
+ - text: ":"
831
+ - generic: Conclusion
832
+ - generic:
833
+ - generic:
834
+ - generic:
835
+ - text: "-"
836
+ - generic: title
837
+ - text: ":"
838
+ - generic: Unit 2. Frameworks for AI Agents
839
+ - generic:
840
+ - generic:
841
+ - generic:
842
+ - generic: sections
843
+ - text: ":"
844
+ - generic:
845
+ - generic:
846
+ - generic:
847
+ - text: "-"
848
+ - generic: local
849
+ - text: ":"
850
+ - generic: unit2/introduction
851
+ - generic:
852
+ - generic:
853
+ - generic:
854
+ - generic: title
855
+ - text: ":"
856
+ - generic: Frameworks for AI Agents
857
+ - generic:
858
+ - generic:
859
+ - generic:
860
+ - text: "-"
861
+ - generic: title
862
+ - text: ":"
863
+ - generic: Unit 2.1 The smolagents framework
864
+ - generic:
865
+ - generic:
866
+ - generic:
867
+ - generic: sections
868
+ - text: ":"
869
+ - generic:
870
+ - generic:
871
+ - generic:
872
+ - text: "-"
873
+ - generic: local
874
+ - text: ":"
875
+ - generic: unit2/smolagents/introduction
876
+ - generic:
877
+ - generic:
878
+ - generic:
879
+ - generic: title
880
+ - text: ":"
881
+ - generic: Introduction to smolagents
882
+ - generic:
883
+ - generic:
884
+ - generic:
885
+ - text: "-"
886
+ - generic: local
887
+ - text: ":"
888
+ - generic: unit2/smolagents/why_use_smolagents
889
+ - generic:
890
+ - generic:
891
+ - generic:
892
+ - generic: title
893
+ - text: ":"
894
+ - generic: Why use smolagents?
895
+ - generic:
896
+ - generic:
897
+ - generic:
898
+ - text: "-"
899
+ - generic: local
900
+ - text: ":"
901
+ - generic: unit2/smolagents/quiz1
902
+ - generic:
903
+ - generic:
904
+ - generic:
905
+ - generic: title
906
+ - text: ":"
907
+ - generic: Quick Quiz 1
908
+ - generic:
909
+ - generic:
910
+ - generic:
911
+ - text: "-"
912
+ - generic: local
913
+ - text: ":"
914
+ - generic: unit2/smolagents/code_agents
915
+ - generic:
916
+ - generic:
917
+ - generic:
918
+ - generic: title
919
+ - text: ":"
920
+ - generic: Building Agents That Use Code
921
+ - generic:
922
+ - generic:
923
+ - generic:
924
+ - text: "-"
925
+ - generic: local
926
+ - text: ":"
927
+ - generic: unit2/smolagents/tool_calling_agents
928
+ - generic:
929
+ - generic:
930
+ - generic:
931
+ - generic: title
932
+ - text: ":"
933
+ - generic: Writing actions as code snippets or JSON blobs
934
+ - generic:
935
+ - generic:
936
+ - generic:
937
+ - text: "-"
938
+ - generic: local
939
+ - text: ":"
940
+ - generic: unit2/smolagents/tools
941
+ - generic:
942
+ - generic:
943
+ - generic:
944
+ - generic: title
945
+ - text: ":"
946
+ - generic: Tools
947
+ - generic:
948
+ - generic:
949
+ - generic:
950
+ - text: "-"
951
+ - generic: local
952
+ - text: ":"
953
+ - generic: unit2/smolagents/retrieval_agents
954
+ - generic:
955
+ - generic:
956
+ - generic:
957
+ - generic: title
958
+ - text: ":"
959
+ - generic: Retrieval Agents
960
+ - generic:
961
+ - generic:
962
+ - generic:
963
+ - text: "-"
964
+ - generic: local
965
+ - text: ":"
966
+ - generic: unit2/smolagents/quiz2
967
+ - generic:
968
+ - generic:
969
+ - generic:
970
+ - generic: title
971
+ - text: ":"
972
+ - generic: Quick Quiz 2
973
+ - generic:
974
+ - generic:
975
+ - generic:
976
+ - text: "-"
977
+ - generic: local
978
+ - text: ":"
979
+ - generic: unit2/smolagents/multi_agent_systems
980
+ - generic:
981
+ - generic:
982
+ - generic:
983
+ - generic: title
984
+ - text: ":"
985
+ - generic: Multi-Agent Systems
986
+ - generic:
987
+ - generic:
988
+ - generic:
989
+ - text: "-"
990
+ - generic: local
991
+ - text: ":"
992
+ - generic: unit2/smolagents/vision_agents
993
+ - generic:
994
+ - generic:
995
+ - generic:
996
+ - generic: title
997
+ - text: ":"
998
+ - generic: Vision and Browser agents
999
+ - generic:
1000
+ - generic:
1001
+ - generic:
1002
+ - text: "-"
1003
+ - generic: local
1004
+ - text: ":"
1005
+ - generic: unit2/smolagents/final_quiz
1006
+ - generic:
1007
+ - generic:
1008
+ - generic:
1009
+ - generic: title
1010
+ - text: ":"
1011
+ - generic: Final Quiz
1012
+ - generic:
1013
+ - generic:
1014
+ - generic:
1015
+ - text: "-"
1016
+ - generic: local
1017
+ - text: ":"
1018
+ - generic: unit2/smolagents/conclusion
1019
+ - generic:
1020
+ - generic:
1021
+ - generic:
1022
+ - generic: title
1023
+ - text: ":"
1024
+ - generic: Conclusion
1025
+ - generic:
1026
+ - generic:
1027
+ - generic:
1028
+ - text: "-"
1029
+ - generic: title
1030
+ - text: ":"
1031
+ - generic: Unit 2.2 The LlamaIndex framework
1032
+ - generic:
1033
+ - generic:
1034
+ - generic:
1035
+ - generic: sections
1036
+ - text: ":"
1037
+ - generic:
1038
+ - generic:
1039
+ - generic:
1040
+ - text: "-"
1041
+ - generic: local
1042
+ - text: ":"
1043
+ - generic: unit2/llama-index/introduction
1044
+ - generic:
1045
+ - generic:
1046
+ - generic:
1047
+ - generic: title
1048
+ - text: ":"
1049
+ - generic: Introduction to LLamaIndex
1050
+ - generic:
1051
+ - generic:
1052
+ - generic:
1053
+ - text: "-"
1054
+ - generic: local
1055
+ - text: ":"
1056
+ - generic: unit2/llama-index/llama-hub
1057
+ - generic:
1058
+ - generic:
1059
+ - generic:
1060
+ - generic: title
1061
+ - text: ":"
1062
+ - generic: Introduction to LlamaHub
1063
+ - generic:
1064
+ - generic:
1065
+ - generic:
1066
+ - text: "-"
1067
+ - generic: local
1068
+ - text: ":"
1069
+ - generic: unit2/llama-index/components
1070
+ - generic:
1071
+ - generic:
1072
+ - generic:
1073
+ - generic: title
1074
+ - text: ":"
1075
+ - generic: What are Components in LlamaIndex?
1076
+ - generic:
1077
+ - generic:
1078
+ - generic:
1079
+ - text: "-"
1080
+ - generic: local
1081
+ - text: ":"
1082
+ - generic: unit2/llama-index/tools
1083
+ - generic:
1084
+ - generic:
1085
+ - generic:
1086
+ - generic: title
1087
+ - text: ":"
1088
+ - generic: Using Tools in LlamaIndex
1089
+ - generic:
1090
+ - generic:
1091
+ - generic:
1092
+ - text: "-"
1093
+ - generic: local
1094
+ - text: ":"
1095
+ - generic: unit2/llama-index/quiz1
1096
+ - generic:
1097
+ - generic:
1098
+ - generic:
1099
+ - generic: title
1100
+ - text: ":"
1101
+ - generic: Quick Quiz 1
1102
+ - generic:
1103
+ - generic:
1104
+ - generic:
1105
+ - text: "-"
1106
+ - generic: local
1107
+ - text: ":"
1108
+ - generic: unit2/llama-index/agents
1109
+ - generic:
1110
+ - generic:
1111
+ - generic:
1112
+ - generic: title
1113
+ - text: ":"
1114
+ - generic: Using Agents in LlamaIndex
1115
+ - generic:
1116
+ - generic:
1117
+ - generic:
1118
+ - text: "-"
1119
+ - generic: local
1120
+ - text: ":"
1121
+ - generic: unit2/llama-index/workflows
1122
+ - generic:
1123
+ - generic:
1124
+ - generic:
1125
+ - generic: title
1126
+ - text: ":"
1127
+ - generic: Creating Agentic Workflows in LlamaIndex
1128
+ - generic:
1129
+ - generic:
1130
+ - generic:
1131
+ - text: "-"
1132
+ - generic: local
1133
+ - text: ":"
1134
+ - generic: unit2/llama-index/quiz2
1135
+ - generic:
1136
+ - generic:
1137
+ - generic:
1138
+ - generic: title
1139
+ - text: ":"
1140
+ - generic: Quick Quiz 2
1141
+ - generic:
1142
+ - generic:
1143
+ - generic:
1144
+ - text: "-"
1145
+ - generic: local
1146
+ - text: ":"
1147
+ - generic: unit2/llama-index/conclusion
1148
+ - generic:
1149
+ - generic:
1150
+ - generic:
1151
+ - generic: title
1152
+ - text: ":"
1153
+ - generic: Conclusion
1154
+ - generic:
1155
+ - generic:
1156
+ - generic:
1157
+ - text: "-"
1158
+ - generic: title
1159
+ - text: ":"
1160
+ - generic: Unit 2.3 The LangGraph framework
1161
+ - generic:
1162
+ - generic:
1163
+ - generic:
1164
+ - generic: sections
1165
+ - text: ":"
1166
+ - generic:
1167
+ - generic:
1168
+ - generic:
1169
+ - text: "-"
1170
+ - generic: local
1171
+ - text: ":"
1172
+ - generic: unit2/langgraph/introduction
1173
+ - generic:
1174
+ - generic:
1175
+ - generic:
1176
+ - generic: title
1177
+ - text: ":"
1178
+ - generic: Introduction to LangGraph
1179
+ - generic:
1180
+ - generic:
1181
+ - generic:
1182
+ - text: "-"
1183
+ - generic: local
1184
+ - text: ":"
1185
+ - generic: unit2/langgraph/when_to_use_langgraph
1186
+ - generic:
1187
+ - generic:
1188
+ - generic:
1189
+ - generic: title
1190
+ - text: ":"
1191
+ - generic: What is LangGraph?
1192
+ - generic:
1193
+ - generic:
1194
+ - generic:
1195
+ - text: "-"
1196
+ - generic: local
1197
+ - text: ":"
1198
+ - generic: unit2/langgraph/building_blocks
1199
+ - generic:
1200
+ - generic:
1201
+ - generic:
1202
+ - generic: title
1203
+ - text: ":"
1204
+ - generic: Building Blocks of LangGraph
1205
+ - generic:
1206
+ - generic:
1207
+ - generic:
1208
+ - text: "-"
1209
+ - generic: local
1210
+ - text: ":"
1211
+ - generic: unit2/langgraph/first_graph
1212
+ - generic:
1213
+ - generic:
1214
+ - generic:
1215
+ - generic: title
1216
+ - text: ":"
1217
+ - generic: Building Your First LangGraph
1218
+ - generic:
1219
+ - generic:
1220
+ - generic:
1221
+ - text: "-"
1222
+ - generic: local
1223
+ - text: ":"
1224
+ - generic: unit2/langgraph/document_analysis_agent
1225
+ - generic:
1226
+ - generic:
1227
+ - generic:
1228
+ - generic: title
1229
+ - text: ":"
1230
+ - generic: Document Analysis Graph
1231
+ - generic:
1232
+ - generic:
1233
+ - generic:
1234
+ - text: "-"
1235
+ - generic: local
1236
+ - text: ":"
1237
+ - generic: unit2/langgraph/quiz1
1238
+ - generic:
1239
+ - generic:
1240
+ - generic:
1241
+ - generic: title
1242
+ - text: ":"
1243
+ - generic: Quick Quiz 1
1244
+ - generic:
1245
+ - generic:
1246
+ - generic:
1247
+ - text: "-"
1248
+ - generic: local
1249
+ - text: ":"
1250
+ - generic: unit2/langgraph/conclusion
1251
+ - generic:
1252
+ - generic:
1253
+ - generic:
1254
+ - generic: title
1255
+ - text: ":"
1256
+ - generic: Conclusion
1257
+ - generic:
1258
+ - generic:
1259
+ - generic:
1260
+ - text: "-"
1261
+ - generic: title
1262
+ - text: ":"
1263
+ - generic: Unit 3. Use Case for Agentic RAG
1264
+ - generic:
1265
+ - generic:
1266
+ - generic:
1267
+ - generic: sections
1268
+ - text: ":"
1269
+ - generic:
1270
+ - generic:
1271
+ - generic:
1272
+ - text: "-"
1273
+ - generic: local
1274
+ - text: ":"
1275
+ - generic: unit3/agentic-rag/introduction
1276
+ - generic:
1277
+ - generic:
1278
+ - generic:
1279
+ - generic: title
1280
+ - text: ":"
1281
+ - generic: Introduction to Use Case for Agentic RAG
1282
+ - generic:
1283
+ - generic:
1284
+ - generic:
1285
+ - text: "-"
1286
+ - generic: local
1287
+ - text: ":"
1288
+ - generic: unit3/agentic-rag/agentic-rag
1289
+ - generic:
1290
+ - generic:
1291
+ - generic:
1292
+ - generic: title
1293
+ - text: ":"
1294
+ - generic: Agentic Retrieval Augmented Generation (RAG)
1295
+ - generic:
1296
+ - generic:
1297
+ - generic:
1298
+ - text: "-"
1299
+ - generic: local
1300
+ - text: ":"
1301
+ - generic: unit3/agentic-rag/invitees
1302
+ - generic:
1303
+ - generic:
1304
+ - generic:
1305
+ - generic: title
1306
+ - text: ":"
1307
+ - generic: Creating a RAG Tool for Guest Stories
1308
+ - generic:
1309
+ - generic:
1310
+ - generic:
1311
+ - text: "-"
1312
+ - generic: local
1313
+ - text: ":"
1314
+ - generic: unit3/agentic-rag/tools
1315
+ - generic:
1316
+ - generic:
1317
+ - generic:
1318
+ - generic: title
1319
+ - text: ":"
1320
+ - generic: Building and Integrating Tools for Your Agent
1321
+ - generic:
1322
+ - generic:
1323
+ - generic:
1324
+ - text: "-"
1325
+ - generic: local
1326
+ - text: ":"
1327
+ - generic: unit3/agentic-rag/agent
1328
+ - generic:
1329
+ - generic:
1330
+ - generic:
1331
+ - generic: title
1332
+ - text: ":"
1333
+ - generic: Creating Your Gala Agent
1334
+ - generic:
1335
+ - generic:
1336
+ - generic:
1337
+ - text: "-"
1338
+ - generic: local
1339
+ - text: ":"
1340
+ - generic: unit3/agentic-rag/conclusion
1341
+ - generic:
1342
+ - generic:
1343
+ - generic:
1344
+ - generic: title
1345
+ - text: ":"
1346
+ - generic: Conclusion
1347
+ - generic:
1348
+ - generic:
1349
+ - generic:
1350
+ - text: "-"
1351
+ - generic: title
1352
+ - text: ":"
1353
+ - generic: Unit 4. Final Project - Create, Test, and Certify Your Agent
1354
+ - generic:
1355
+ - generic:
1356
+ - generic:
1357
+ - generic: sections
1358
+ - text: ":"
1359
+ - generic:
1360
+ - generic:
1361
+ - generic:
1362
+ - text: "-"
1363
+ - generic: local
1364
+ - text: ":"
1365
+ - generic: unit4/introduction
1366
+ - generic:
1367
+ - generic:
1368
+ - generic:
1369
+ - generic: title
1370
+ - text: ":"
1371
+ - generic: Introduction to the Final Unit
1372
+ - generic:
1373
+ - generic:
1374
+ - generic:
1375
+ - text: "-"
1376
+ - generic: local
1377
+ - text: ":"
1378
+ - generic: unit4/what-is-gaia
1379
+ - generic:
1380
+ - generic:
1381
+ - generic:
1382
+ - generic: title
1383
+ - text: ":"
1384
+ - generic: What is GAIA?
1385
+ - generic:
1386
+ - generic:
1387
+ - generic:
1388
+ - text: "-"
1389
+ - generic: local
1390
+ - text: ":"
1391
+ - generic: unit4/hands-on
1392
+ - generic:
1393
+ - generic:
1394
+ - generic:
1395
+ - generic: title
1396
+ - text: ":"
1397
+ - generic: The Final Hands-On
1398
+ - generic:
1399
+ - generic:
1400
+ - generic:
1401
+ - text: "-"
1402
+ - generic: local
1403
+ - text: ":"
1404
+ - generic: unit4/get-your-certificate
1405
+ - generic:
1406
+ - generic:
1407
+ - generic:
1408
+ - generic: title
1409
+ - text: ":"
1410
+ - generic: Get Your Certificate Of Excellence
1411
+ - generic:
1412
+ - generic:
1413
+ - generic:
1414
+ - text: "-"
1415
+ - generic: local
1416
+ - text: ":"
1417
+ - generic: unit4/conclusion
1418
+ - generic:
1419
+ - generic:
1420
+ - generic:
1421
+ - generic: title
1422
+ - text: ":"
1423
+ - generic: Conclusion of the Course
1424
+ - generic:
1425
+ - generic:
1426
+ - generic:
1427
+ - text: "-"
1428
+ - generic: local
1429
+ - text: ":"
1430
+ - generic: unit4/additional-readings
1431
+ - generic:
1432
+ - generic:
1433
+ - generic:
1434
+ - generic: title
1435
+ - text: ":"
1436
+ - generic: What Should You Learn Now?
1437
+ - generic:
1438
+ - generic:
1439
+ - generic:
1440
+ - text: "-"
1441
+ - generic: title
1442
+ - text: ":"
1443
+ - generic: Bonus Unit 1. Fine-tuning an LLM for Function-calling
1444
+ - generic:
1445
+ - generic:
1446
+ - generic:
1447
+ - generic: sections
1448
+ - text: ":"
1449
+ - generic:
1450
+ - generic:
1451
+ - generic:
1452
+ - text: "-"
1453
+ - generic: local
1454
+ - text: ":"
1455
+ - generic: bonus-unit1/introduction
1456
+ - generic:
1457
+ - generic:
1458
+ - generic:
1459
+ - generic: title
1460
+ - text: ":"
1461
+ - generic: Introduction
1462
+ - generic:
1463
+ - generic:
1464
+ - generic:
1465
+ - text: "-"
1466
+ - generic: local
1467
+ - text: ":"
1468
+ - generic: bonus-unit1/what-is-function-calling
1469
+ - generic:
1470
+ - generic:
1471
+ - generic:
1472
+ - generic: title
1473
+ - text: ":"
1474
+ - generic: What is Function Calling?
1475
+ - generic:
1476
+ - generic:
1477
+ - generic:
1478
+ - text: "-"
1479
+ - generic: local
1480
+ - text: ":"
1481
+ - generic: bonus-unit1/fine-tuning
1482
+ - generic:
1483
+ - generic:
1484
+ - generic:
1485
+ - generic: title
1486
+ - text: ":"
1487
+ - generic: Let's Fine-Tune your model for Function-calling
1488
+ - generic:
1489
+ - generic:
1490
+ - generic:
1491
+ - text: "-"
1492
+ - generic: local
1493
+ - text: ":"
1494
+ - generic: bonus-unit1/conclusion
1495
+ - generic:
1496
+ - generic:
1497
+ - generic:
1498
+ - generic: title
1499
+ - text: ":"
1500
+ - generic: Conclusion
1501
+ - generic:
1502
+ - generic:
1503
+ - generic:
1504
+ - text: "-"
1505
+ - generic: title
1506
+ - text: ":"
1507
+ - generic: Bonus Unit 2. Agent Observability and Evaluation
1508
+ - generic:
1509
+ - generic:
1510
+ - generic:
1511
+ - generic: sections
1512
+ - text: ":"
1513
+ - generic:
1514
+ - generic:
1515
+ - generic:
1516
+ - text: "-"
1517
+ - generic: local
1518
+ - text: ":"
1519
+ - generic: bonus-unit2/introduction
1520
+ - generic:
1521
+ - generic:
1522
+ - generic:
1523
+ - generic: title
1524
+ - text: ":"
1525
+ - generic: Introduction
1526
+ - generic:
1527
+ - generic:
1528
+ - generic:
1529
+ - text: "-"
1530
+ - generic: local
1531
+ - text: ":"
1532
+ - generic: bonus-unit2/what-is-agent-observability-and-evaluation
1533
+ - generic:
1534
+ - generic:
1535
+ - generic:
1536
+ - generic: title
1537
+ - text: ":"
1538
+ - generic: What is agent observability and evaluation?
1539
+ - generic:
1540
+ - generic:
1541
+ - generic:
1542
+ - text: "-"
1543
+ - generic: local
1544
+ - text: ":"
1545
+ - generic: bonus-unit2/monitoring-and-evaluating-agents-notebook
1546
+ - generic:
1547
+ - generic:
1548
+ - generic:
1549
+ - generic: title
1550
+ - text: ":"
1551
+ - generic: Monitoring and evaluating agents
1552
+ - generic:
1553
+ - generic:
1554
+ - generic:
1555
+ - text: "-"
1556
+ - generic: local
1557
+ - text: ":"
1558
+ - generic: bonus-unit2/quiz
1559
+ - generic:
1560
+ - generic:
1561
+ - generic:
1562
+ - generic: title
1563
+ - text: ":"
1564
+ - generic: Quiz
1565
+ - generic:
1566
+ - generic:
1567
+ - generic:
1568
+ - text: "-"
1569
+ - generic: title
1570
+ - text: ":"
1571
+ - generic: Bonus Unit 3. Agents in Games with Pokemon
1572
+ - generic:
1573
+ - generic:
1574
+ - generic:
1575
+ - generic: sections
1576
+ - text: ":"
1577
+ - generic:
1578
+ - generic:
1579
+ - generic:
1580
+ - text: "-"
1581
+ - generic: local
1582
+ - text: ":"
1583
+ - generic: bonus-unit3/introduction
1584
+ - generic:
1585
+ - generic:
1586
+ - generic:
1587
+ - generic: title
1588
+ - text: ":"
1589
+ - generic: Introduction
1590
+ - generic:
1591
+ - generic:
1592
+ - generic:
1593
+ - text: "-"
1594
+ - generic: local
1595
+ - text: ":"
1596
+ - generic: bonus-unit3/state-of-art
1597
+ - generic:
1598
+ - generic:
1599
+ - generic:
1600
+ - generic: title
1601
+ - text: ":"
1602
+ - generic: The State of the Art in Using LLMs in Games
1603
+ - generic:
1604
+ - generic:
1605
+ - generic:
1606
+ - text: "-"
1607
+ - generic: local
1608
+ - text: ":"
1609
+ - generic: bonus-unit3/from-llm-to-agents
1610
+ - generic:
1611
+ - generic:
1612
+ - generic:
1613
+ - generic: title
1614
+ - text: ":"
1615
+ - generic: From LLMs to AI Agents
1616
+ - generic:
1617
+ - generic:
1618
+ - generic:
1619
+ - text: "-"
1620
+ - generic: local
1621
+ - text: ":"
1622
+ - generic: bonus-unit3/building_your_pokemon_agent
1623
+ - generic:
1624
+ - generic:
1625
+ - generic:
1626
+ - generic: title
1627
+ - text: ":"
1628
+ - generic: Build Your Own Pokémon Battle Agent
1629
+ - generic:
1630
+ - generic:
1631
+ - generic:
1632
+ - text: "-"
1633
+ - generic: local
1634
+ - text: ":"
1635
+ - generic: bonus-unit3/launching_agent_battle
1636
+ - generic:
1637
+ - generic:
1638
+ - generic:
1639
+ - generic: title
1640
+ - text: ":"
1641
+ - generic: Launching Your Pokémon Battle Agent
1642
+ - generic:
1643
+ - generic:
1644
+ - generic:
1645
+ - text: "-"
1646
+ - generic: local
1647
+ - text: ":"
1648
+ - generic: bonus-unit3/conclusion
1649
+ - generic:
1650
+ - generic:
1651
+ - generic:
1652
+ - generic: title
1653
+ - text: ":"
1654
+ - generic: Conclusion
1655
+ - alert [ref=e818]
.pre-commit-config.yaml ADDED
@@ -0,0 +1,9 @@
 
 
 
 
 
 
 
 
 
 
1
+ repos:
2
+ - repo: local
3
+ hooks:
4
+ - id: pylint
5
+ name: PyLint
6
+ entry: python3 -m pylint.__main__
7
+ language: system
8
+ files: \.py$
9
+ require_serial: true
.pylintrc ADDED
@@ -0,0 +1,21 @@
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
1
+ [MASTER]
2
+ init-hook='import sys; sys.path.append(".")'
3
+ ignore-patterns=test_.*?py,__.*?py,conftest.py,tid_.*?_check.py,setup.py
4
+ ignore=Dockerfile,README.md,requirements.txt,CODEOWNERS
5
+ ignore-paths=mcpuniverse/app/db
6
+
7
+ [FORMAT]
8
+ # Maximum number of characters on a single line.
9
+ max-line-length=120
10
+
11
+ [MESSAGES CONTROL]
12
+ disable=no-member,import-error,no-self-use,useless-option-value,inconsistent-return-statements,too-many-positional-arguments
13
+
14
+ [DESIGN]
15
+ max-statements=75
16
+ max-locals=35
17
+ max-branches=25
18
+ max-args=20
19
+ max-nested-blocks=6
20
+ max-attributes=15
21
+ min-similarity-lines=200
AI_ETHICS.md ADDED
@@ -0,0 +1,13 @@
 
 
 
 
 
 
 
 
 
 
 
 
 
 
1
+ ## Ethics disclaimer for Salesforce AI models, data, code
2
+
3
+ This release is for research purposes only in support of an academic
4
+ paper. Our models, datasets, and code are not specifically designed or
5
+ evaluated for all downstream purposes. We strongly recommend users
6
+ evaluate and address potential concerns related to accuracy, safety, and
7
+ fairness before deploying this model. We encourage users to consider the
8
+ common limitations of AI, comply with applicable laws, and leverage best
9
+ practices when selecting use cases, particularly for high-risk scenarios
10
+ where errors or misuse could significantly impact people’s lives, rights,
11
+ or safety. For further guidance on use cases, refer to our standard
12
+ [AUP](https://www.salesforce.com/content/dam/web/en_us/www/documents/legal/Agreements/policies/ExternalFacing_Services_Policy.pdf)
13
+ and [AI AUP](https://www.salesforce.com/content/dam/web/en_us/www/documents/legal/Agreements/policies/ai-acceptable-use-policy.pdf).
CODEOWNERS ADDED
@@ -0,0 +1,3 @@
 
 
 
 
1
+ # Comment line immediately above ownership line is reserved for related other information. Please be careful while editing.
2
+ #ECCN:Open Source
3
+ #GUSINFO:Open Source,Open Source Workflow
CODE_OF_CONDUCT.md ADDED
@@ -0,0 +1,105 @@
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
1
+ # Salesforce Open Source Community Code of Conduct
2
+
3
+ ## About the Code of Conduct
4
+
5
+ Equality is a core value at Salesforce. We believe a diverse and inclusive
6
+ community fosters innovation and creativity, and are committed to building a
7
+ culture where everyone feels included.
8
+
9
+ Salesforce open-source projects are committed to providing a friendly, safe, and
10
+ welcoming environment for all, regardless of gender identity and expression,
11
+ sexual orientation, disability, physical appearance, body size, ethnicity, nationality,
12
+ race, age, religion, level of experience, education, socioeconomic status, or
13
+ other similar personal characteristics.
14
+
15
+ The goal of this code of conduct is to specify a baseline standard of behavior so
16
+ that people with different social values and communication styles can work
17
+ together effectively, productively, and respectfully in our open source community.
18
+ It also establishes a mechanism for reporting issues and resolving conflicts.
19
+
20
+ All questions and reports of abusive, harassing, or otherwise unacceptable behavior
21
+ in a Salesforce open-source project may be reported by contacting the Salesforce
22
+ Open Source Conduct Committee at ossconduct@salesforce.com.
23
+
24
+ ## Our Pledge
25
+
26
+ In the interest of fostering an open and welcoming environment, we as
27
+ contributors and maintainers pledge to making participation in our project and
28
+ our community a harassment-free experience for everyone, regardless of gender
29
+ identity and expression, sexual orientation, disability, physical appearance,
30
+ body size, ethnicity, nationality, race, age, religion, level of experience, education,
31
+ socioeconomic status, or other similar personal characteristics.
32
+
33
+ ## Our Standards
34
+
35
+ Examples of behavior that contributes to creating a positive environment
36
+ include:
37
+
38
+ * Using welcoming and inclusive language
39
+ * Being respectful of differing viewpoints and experiences
40
+ * Gracefully accepting constructive criticism
41
+ * Focusing on what is best for the community
42
+ * Showing empathy toward other community members
43
+
44
+ Examples of unacceptable behavior by participants include:
45
+
46
+ * The use of sexualized language or imagery and unwelcome sexual attention or
47
+ advances
48
+ * Personal attacks, insulting/derogatory comments, or trolling
49
+ * Public or private harassment
50
+ * Publishing, or threatening to publish, others' private information—such as
51
+ a physical or electronic address—without explicit permission
52
+ * Other conduct which could reasonably be considered inappropriate in a
53
+ professional setting
54
+ * Advocating for or encouraging any of the above behaviors
55
+
56
+ ## Our Responsibilities
57
+
58
+ Project maintainers are responsible for clarifying the standards of acceptable
59
+ behavior and are expected to take appropriate and fair corrective action in
60
+ response to any instances of unacceptable behavior.
61
+
62
+ Project maintainers have the right and responsibility to remove, edit, or
63
+ reject comments, commits, code, wiki edits, issues, and other contributions
64
+ that are not aligned with this Code of Conduct, or to ban temporarily or
65
+ permanently any contributor for other behaviors that they deem inappropriate,
66
+ threatening, offensive, or harmful.
67
+
68
+ ## Scope
69
+
70
+ This Code of Conduct applies both within project spaces and in public spaces
71
+ when an individual is representing the project or its community. Examples of
72
+ representing a project or community include using an official project email
73
+ address, posting via an official social media account, or acting as an appointed
74
+ representative at an online or offline event. Representation of a project may be
75
+ further defined and clarified by project maintainers.
76
+
77
+ ## Enforcement
78
+
79
+ Instances of abusive, harassing, or otherwise unacceptable behavior may be
80
+ reported by contacting the Salesforce Open Source Conduct Committee
81
+ at ossconduct@salesforce.com. All complaints will be reviewed and investigated
82
+ and will result in a response that is deemed necessary and appropriate to the
83
+ circumstances. The committee is obligated to maintain confidentiality with
84
+ regard to the reporter of an incident. Further details of specific enforcement
85
+ policies may be posted separately.
86
+
87
+ Project maintainers who do not follow or enforce the Code of Conduct in good
88
+ faith may face temporary or permanent repercussions as determined by other
89
+ members of the project's leadership and the Salesforce Open Source Conduct
90
+ Committee.
91
+
92
+ ## Attribution
93
+
94
+ This Code of Conduct is adapted from the [Contributor Covenant][contributor-covenant-home],
95
+ version 1.4, available at https://www.contributor-covenant.org/version/1/4/code-of-conduct.html.
96
+ It includes adaptions and additions from [Go Community Code of Conduct][golang-coc],
97
+ [CNCF Code of Conduct][cncf-coc], and [Microsoft Open Source Code of Conduct][microsoft-coc].
98
+
99
+ This Code of Conduct is licensed under the [Creative Commons Attribution 3.0 License][cc-by-3-us].
100
+
101
+ [contributor-covenant-home]: https://www.contributor-covenant.org (https://www.contributor-covenant.org/)
102
+ [golang-coc]: https://golang.org/conduct
103
+ [cncf-coc]: https://github.com/cncf/foundation/blob/master/code-of-conduct.md
104
+ [microsoft-coc]: https://opensource.microsoft.com/codeofconduct/
105
+ [cc-by-3-us]: https://creativecommons.org/licenses/by/3.0/us/
CONTRIBUTING.md ADDED
@@ -0,0 +1,87 @@
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
1
+ # Contributing Guide For MCP-Universe
2
+
3
+ This page lists the operational governance model of this project, as well as the recommendations and requirements for how to best contribute to MCPWorld. We strive to obey these as best as possible. As always, thanks for contributing – we hope these guidelines make it easier and shed some light on our approach and processes.
4
+
5
+ # Governance Model
6
+
7
+ ## Published but not supported
8
+
9
+ The intent and goal of open sourcing this project is because it may contain useful or interesting code/concepts that we wish to share with the larger open source community. Although occasional work may be done on it, we will not be looking for or soliciting contributions.
10
+
11
+ # Getting started
12
+
13
+ Please join the community on {Here list Slack channels, Email lists, Glitter, Discord, etc... links}. Also please make sure to take a look at the project [roadmap](ROADMAP.md) to see where are headed.
14
+
15
+ # Issues, requests & ideas
16
+
17
+ Use GitHub Issues page to submit issues, enhancement requests and discuss ideas.
18
+
19
+ ### Bug Reports and Fixes
20
+ - If you find a bug, please search for it in the [Issues](https://github.com/{project_slug}/issues), and if it isn't already tracked,
21
+ [create a new issue](https://github.com/{project_slug}/issues/new). Fill out the "Bug Report" section of the issue template. Even if an Issue is closed, feel free to comment and add details, it will still
22
+ be reviewed.
23
+ - Issues that have already been identified as a bug (note: able to reproduce) will be labelled `bug`.
24
+ - If you'd like to submit a fix for a bug, [send a Pull Request](#creating_a_pull_request) and mention the Issue number.
25
+ - Include tests that isolate the bug and verifies that it was fixed.
26
+
27
+ ### New Features
28
+ - If you'd like to add new functionality to this project, describe the problem you want to solve in a [new Issue](https://github.com/{project_slug}/issues/new).
29
+ - Issues that have been identified as a feature request will be labelled `enhancement`.
30
+ - If you'd like to implement the new feature, please wait for feedback from the project
31
+ maintainers before spending too much time writing the code. In some cases, `enhancement`s may
32
+ not align well with the project objectives at the time.
33
+
34
+ ### Tests, Documentation, Miscellaneous
35
+ - If you'd like to improve the tests, you want to make the documentation clearer, you have an
36
+ alternative implementation of something that may have advantages over the way its currently
37
+ done, or you have any other change, we would be happy to hear about it!
38
+ - If its a trivial change, go ahead and [send a Pull Request](#creating_a_pull_request) with the changes you have in mind.
39
+ - If not, [open an Issue](https://github.com/{project_slug}/issues/new) to discuss the idea first.
40
+
41
+ If you're new to our project and looking for some way to make your first contribution, look for
42
+ Issues labelled `good first contribution`.
43
+
44
+ # Contribution Checklist
45
+
46
+ - [x] Clean, simple, well styled code
47
+ - [x] Commits should be atomic and messages must be descriptive. Related issues should be mentioned by Issue number.
48
+ - [x] Comments
49
+ - Module-level & function-level comments.
50
+ - Comments on complex blocks of code or algorithms (include references to sources).
51
+ - [x] Tests
52
+ - The test suite, if provided, must be complete and pass
53
+ - Increase code coverage, not versa.
54
+ - Use any of our testkits that contains a bunch of testing facilities you would need. For example: `import com.salesforce.op.test._` and borrow inspiration from existing tests.
55
+ - [x] Dependencies
56
+ - Minimize number of dependencies.
57
+ - Prefer Apache 2.0, BSD3, MIT, ISC and MPL licenses.
58
+ - [x] Reviews
59
+ - Changes must be approved via peer code review
60
+
61
+ # Creating a Pull Request
62
+
63
+ 1. **Ensure the bug/feature was not already reported** by searching on GitHub under Issues. If none exists, create a new issue so that other contributors can keep track of what you are trying to add/fix and offer suggestions (or let you know if there is already an effort in progress).
64
+ 3. **Clone** the forked repo to your machine.
65
+ 4. **Create** a new branch to contain your work (e.g. `git br fix-issue-11`)
66
+ 4. **Commit** changes to your own branch.
67
+ 5. **Push** your work back up to your fork. (e.g. `git push fix-issue-11`)
68
+ 6. **Submit** a Pull Request against the `main` branch and refer to the issue(s) you are fixing. Try not to pollute your pull request with unintended changes. Keep it simple and small.
69
+ 7. **Sign** the Salesforce CLA (you will be prompted to do so when submitting the Pull Request)
70
+
71
+ > **NOTE**: Be sure to [sync your fork](https://help.github.com/articles/syncing-a-fork/) before making a pull request.
72
+
73
+ # Contributor License Agreement ("CLA")
74
+ In order to accept your pull request, we need you to submit a CLA. You only need
75
+ to do this once to work on any of Salesforce's open source projects.
76
+
77
+ Complete your CLA here: <https://cla.salesforce.com/sign-cla>
78
+
79
+ # Issues
80
+ We use GitHub issues to track public bugs. Please ensure your description is
81
+ clear and has sufficient instructions to be able to reproduce the issue.
82
+
83
+ # Code of Conduct
84
+ Please follow our [Code of Conduct](CODE_OF_CONDUCT.md).
85
+
86
+ # License
87
+ By contributing your code, you agree to license your contribution under the terms of our project [LICENSE](LICENSE.txt) and to sign the [Salesforce CLA](https://cla.salesforce.com/sign-cla)
LICENSE.txt ADDED
@@ -0,0 +1,207 @@
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
1
+ Apache License Version 2.0
2
+
3
+ Copyright (c) 2024 Salesforce, Inc.
4
+ All rights reserved.
5
+
6
+ Apache License
7
+ Version 2.0, January 2004
8
+ http://www.apache.org/licenses/
9
+
10
+ TERMS AND CONDITIONS FOR USE, REPRODUCTION, AND DISTRIBUTION
11
+
12
+ 1. Definitions.
13
+
14
+ "License" shall mean the terms and conditions for use, reproduction,
15
+ and distribution as defined by Sections 1 through 9 of this document.
16
+
17
+ "Licensor" shall mean the copyright owner or entity authorized by
18
+ the copyright owner that is granting the License.
19
+
20
+ "Legal Entity" shall mean the union of the acting entity and all
21
+ other entities that control, are controlled by, or are under common
22
+ control with that entity. For the purposes of this definition,
23
+ "control" means (i) the power, direct or indirect, to cause the
24
+ direction or management of such entity, whether by contract or
25
+ otherwise, or (ii) ownership of fifty percent (50%) or more of the
26
+ outstanding shares, or (iii) beneficial ownership of such entity.
27
+
28
+ "You" (or "Your") shall mean an individual or Legal Entity
29
+ exercising permissions granted by this License.
30
+
31
+ "Source" form shall mean the preferred form for making modifications,
32
+ including but not limited to software source code, documentation
33
+ source, and configuration files.
34
+
35
+ "Object" form shall mean any form resulting from mechanical
36
+ transformation or translation of a Source form, including but
37
+ not limited to compiled object code, generated documentation,
38
+ and conversions to other media types.
39
+
40
+ "Work" shall mean the work of authorship, whether in Source or
41
+ Object form, made available under the License, as indicated by a
42
+ copyright notice that is included in or attached to the work
43
+ (an example is provided in the Appendix below).
44
+
45
+ "Derivative Works" shall mean any work, whether in Source or Object
46
+ form, that is based on (or derived from) the Work and for which the
47
+ editorial revisions, annotations, elaborations, or other modifications
48
+ represent, as a whole, an original work of authorship. For the purposes
49
+ of this License, Derivative Works shall not include works that remain
50
+ separable from, or merely link (or bind by name) to the interfaces of,
51
+ the Work and Derivative Works thereof.
52
+
53
+ "Contribution" shall mean any work of authorship, including
54
+ the original version of the Work and any modifications or additions
55
+ to that Work or Derivative Works thereof, that is intentionally
56
+ submitted to Licensor for inclusion in the Work by the copyright owner
57
+ or by an individual or Legal Entity authorized to submit on behalf of
58
+ the copyright owner. For the purposes of this definition, "submitted"
59
+ means any form of electronic, verbal, or written communication sent
60
+ to the Licensor or its representatives, including but not limited to
61
+ communication on electronic mailing lists, source code control systems,
62
+ and issue tracking systems that are managed by, or on behalf of, the
63
+ Licensor for the purpose of discussing and improving the Work, but
64
+ excluding communication that is conspicuously marked or otherwise
65
+ designated in writing by the copyright owner as "Not a Contribution."
66
+
67
+ "Contributor" shall mean Licensor and any individual or Legal Entity
68
+ on behalf of whom a Contribution has been received by Licensor and
69
+ subsequently incorporated within the Work.
70
+
71
+ 2. Grant of Copyright License. Subject to the terms and conditions of
72
+ this License, each Contributor hereby grants to You a perpetual,
73
+ worldwide, non-exclusive, no-charge, royalty-free, irrevocable
74
+ copyright license to reproduce, prepare Derivative Works of,
75
+ publicly display, publicly perform, sublicense, and distribute the
76
+ Work and such Derivative Works in Source or Object form.
77
+
78
+ 3. Grant of Patent License. Subject to the terms and conditions of
79
+ this License, each Contributor hereby grants to You a perpetual,
80
+ worldwide, non-exclusive, no-charge, royalty-free, irrevocable
81
+ (except as stated in this section) patent license to make, have made,
82
+ use, offer to sell, sell, import, and otherwise transfer the Work,
83
+ where such license applies only to those patent claims licensable
84
+ by such Contributor that are necessarily infringed by their
85
+ Contribution(s) alone or by combination of their Contribution(s)
86
+ with the Work to which such Contribution(s) was submitted. If You
87
+ institute patent litigation against any entity (including a
88
+ cross-claim or counterclaim in a lawsuit) alleging that the Work
89
+ or a Contribution incorporated within the Work constitutes direct
90
+ or contributory patent infringement, then any patent licenses
91
+ granted to You under this License for that Work shall terminate
92
+ as of the date such litigation is filed.
93
+
94
+ 4. Redistribution. You may reproduce and distribute copies of the
95
+ Work or Derivative Works thereof in any medium, with or without
96
+ modifications, and in Source or Object form, provided that You
97
+ meet the following conditions:
98
+
99
+ (a) You must give any other recipients of the Work or
100
+ Derivative Works a copy of this License; and
101
+
102
+ (b) You must cause any modified files to carry prominent notices
103
+ stating that You changed the files; and
104
+
105
+ (c) You must retain, in the Source form of any Derivative Works
106
+ that You distribute, all copyright, patent, trademark, and
107
+ attribution notices from the Source form of the Work,
108
+ excluding those notices that do not pertain to any part of
109
+ the Derivative Works; and
110
+
111
+ (d) If the Work includes a "NOTICE" text file as part of its
112
+ distribution, then any Derivative Works that You distribute must
113
+ include a readable copy of the attribution notices contained
114
+ within such NOTICE file, excluding those notices that do not
115
+ pertain to any part of the Derivative Works, in at least one
116
+ of the following places: within a NOTICE text file distributed
117
+ as part of the Derivative Works; within the Source form or
118
+ documentation, if provided along with the Derivative Works; or,
119
+ within a display generated by the Derivative Works, if and
120
+ wherever such third-party notices normally appear. The contents
121
+ of the NOTICE file are for informational purposes only and
122
+ do not modify the License. You may add Your own attribution
123
+ notices within Derivative Works that You distribute, alongside
124
+ or as an addendum to the NOTICE text from the Work, provided
125
+ that such additional attribution notices cannot be construed
126
+ as modifying the License.
127
+
128
+ You may add Your own copyright statement to Your modifications and
129
+ may provide additional or different license terms and conditions
130
+ for use, reproduction, or distribution of Your modifications, or
131
+ for any such Derivative Works as a whole, provided Your use,
132
+ reproduction, and distribution of the Work otherwise complies with
133
+ the conditions stated in this License.
134
+
135
+ 5. Submission of Contributions. Unless You explicitly state otherwise,
136
+ any Contribution intentionally submitted for inclusion in the Work
137
+ by You to the Licensor shall be under the terms and conditions of
138
+ this License, without any additional terms or conditions.
139
+ Notwithstanding the above, nothing herein shall supersede or modify
140
+ the terms of any separate license agreement you may have executed
141
+ with Licensor regarding such Contributions.
142
+
143
+ 6. Trademarks. This License does not grant permission to use the trade
144
+ names, trademarks, service marks, or product names of the Licensor,
145
+ except as required for reasonable and customary use in describing the
146
+ origin of the Work and reproducing the content of the NOTICE file.
147
+
148
+ 7. Disclaimer of Warranty. Unless required by applicable law or
149
+ agreed to in writing, Licensor provides the Work (and each
150
+ Contributor provides its Contributions) on an "AS IS" BASIS,
151
+ WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or
152
+ implied, including, without limitation, any warranties or conditions
153
+ of TITLE, NON-INFRINGEMENT, MERCHANTABILITY, or FITNESS FOR A
154
+ PARTICULAR PURPOSE. You are solely responsible for determining the
155
+ appropriateness of using or redistributing the Work and assume any
156
+ risks associated with Your exercise of permissions under this License.
157
+
158
+ 8. Limitation of Liability. In no event and under no legal theory,
159
+ whether in tort (including negligence), contract, or otherwise,
160
+ unless required by applicable law (such as deliberate and grossly
161
+ negligent acts) or agreed to in writing, shall any Contributor be
162
+ liable to You for damages, including any direct, indirect, special,
163
+ incidental, or consequential damages of any character arising as a
164
+ result of this License or out of the use or inability to use the
165
+ Work (including but not limited to damages for loss of goodwill,
166
+ work stoppage, computer failure or malfunction, or any and all
167
+ other commercial damages or losses), even if such Contributor
168
+ has been advised of the possibility of such damages.
169
+
170
+ 9. Accepting Warranty or Additional Liability. While redistributing
171
+ the Work or Derivative Works thereof, You may choose to offer,
172
+ and charge a fee for, acceptance of support, warranty, indemnity,
173
+ or other liability obligations and/or rights consistent with this
174
+ License. However, in accepting such obligations, You may act only
175
+ on Your own behalf and on Your sole responsibility, not on behalf
176
+ of any other Contributor, and only if You agree to indemnify,
177
+ defend, and hold each Contributor harmless for any liability
178
+ incurred by, or claims asserted against, such Contributor by reason
179
+ of your accepting any such warranty or additional liability.
180
+
181
+ END OF TERMS AND CONDITIONS
182
+
183
+ APPENDIX: How to apply the Apache License to your work.
184
+
185
+ To apply the Apache License to your work, attach the following
186
+ boilerplate notice, with the fields enclosed by brackets "{}"
187
+ replaced with your own identifying information. (Don't include
188
+ the brackets!) The text should be enclosed in the appropriate
189
+ comment syntax for the file format. We also recommend that a
190
+ file or class name and description of purpose be included on the
191
+ same "printed page" as the copyright notice for easier
192
+ identification within third-party archives.
193
+
194
+ Copyright {yyyy} {name of copyright owner}
195
+
196
+ Licensed under the Apache License, Version 2.0 (the "License");
197
+ you may not use this file except in compliance with the License.
198
+ You may obtain a copy of the License at
199
+
200
+ http://www.apache.org/licenses/LICENSE-2.0
201
+
202
+ Unless required by applicable law or agreed to in writing, software
203
+ distributed under the License is distributed on an "AS IS" BASIS,
204
+ WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
205
+ See the License for the specific language governing permissions and
206
+ limitations under the License.
207
+
Makefile ADDED
@@ -0,0 +1,55 @@
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
1
+ DB_URL=postgresql://root:secret@localhost:5432/mcpuniverse?sslmode=disable
2
+
3
+ test:
4
+ PYTHONPATH=. pytest tests/
5
+
6
+ sqlc:
7
+ sqlc generate
8
+
9
+ redis:
10
+ docker run --name redis -p 6379:6379 -d redis:7.2-alpine
11
+
12
+ dropredis:
13
+ docker stop redis
14
+ docker container rm redis
15
+
16
+ postgres:
17
+ docker run --name postgres -p 5432:5432 -e POSTGRES_USER=root -e POSTGRES_PASSWORD=secret -d postgres:15.13-alpine
18
+
19
+ droppostgres:
20
+ docker stop postgres
21
+ docker container rm postgres
22
+
23
+ createdb:
24
+ docker exec -it postgres createdb --username=root --owner=root mcpuniverse
25
+
26
+ dropdb:
27
+ docker exec -it postgres dropdb mcpuniverse
28
+
29
+ new_migration:
30
+ migrate create -ext sql -dir mcpuniverse/app/db/migration -seq $(name)
31
+
32
+ migrateup:
33
+ migrate -path mcpuniverse/app/db/migration -database "$(DB_URL)" -verbose up
34
+
35
+ migratedown:
36
+ migrate -path mcpuniverse/app/db/migration -database "$(DB_URL)" -verbose down
37
+
38
+ dashboard:
39
+ PYTHONPATH=. uvicorn mcpuniverse.dashboard.app:app
40
+
41
+ kafka:
42
+ docker run --name kafka -p 9092:9092 -d apache/kafka:4.1.0
43
+
44
+ dropkafka:
45
+ docker stop kafka
46
+ docker container rm kafka
47
+
48
+ rabbitmq:
49
+ docker run --name rabbitmq -p 5672:5672 -p 15672:15672 -d rabbitmq:4.1.4-management
50
+
51
+ droprabbitmq:
52
+ docker stop rabbitmq
53
+ docker container rm rabbitmq
54
+
55
+ .PHONY: test sqlc redis dropredis postgres droppostgres createdb dropdb new_migration migrateup migratedown dashboard kafka dropkafka rabbitmq droprabbitmq
README.md CHANGED
@@ -1,3 +1,622 @@
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
1
  ---
2
- license: apache-2.0
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
3
  ---
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
1
+ # <img src="assets/icon.png" alt="MCP-Universe" width="23" height="23"> MCP-Universe
2
+
3
+ [![Paper](https://img.shields.io/badge/Paper-arXiv:2508.14704-B31B1B?style=for-the-badge&logo=arxiv&logoColor=white)](https://arxiv.org/abs/2508.14704)
4
+ [![Website](https://img.shields.io/badge/Website-Live-4285F4?style=for-the-badge&logo=googlechrome&logoColor=white)](https://mcp-universe.github.io/)
5
+ [![Leaderboard](https://img.shields.io/badge/Leaderboard-Results-FF6B35?style=for-the-badge&logo=chartdotjs&logoColor=white)](https://mcp-universe.github.io/#results)
6
+ [![Discord](https://img.shields.io/badge/Discord-Join_Community-5865F2?style=for-the-badge&logo=discord&logoColor=white)](https://discord.gg/t9tU77GF)
7
+
8
+ ### 🎉 Latest Updates
9
+
10
+ > **📊 [MCPMark Evaluation](#mcpmark-benchmark)** - MCP-Universe now supports evaluating the MCPMark tasks
11
+ >
12
+ > **🚀 [MCP+](#mcp-precision-context-management-for-mcp-agents)** - Agentic wrapper on MCP clients which reduce token costs by up to 75%
13
+ >
14
+ > **🔬 [Deep Research Agent](#deep-research-agent-wide--deep-wd-research)** - Scale the Width of Deep Research Agents with parallel tool calling, improving performance and efficiency
15
+
16
+ </div>
17
+
18
  ---
19
+
20
+ ## What is MCP-Universe?
21
+
22
+ MCP-Universe is a comprehensive ecosystem for building, optimizing, and evaluating AI agents that interact with the Model Context Protocol (MCP). Beyond our industry-leading benchmark for real-world MCP server interactions, MCP-Universe provides production-ready tools for agent development including specialized research agents ([**Deep Research Agent**](#deep-research-agent-wide--deep-wd-research)), intelligent context management ([**MCP+**](#mcp-precision-context-management-for-mcp-agents)), and sophisticated orchestration workflows.
23
+
24
+ <div align="center">
25
+
26
+ ![MCP-Universe Introduction](assets/intro-mcp-universe.png)
27
+
28
+ </div>
29
+
30
+ **Benchmarking:** Unlike existing benchmarks that rely on overly simplistic tasks, MCP-Universe addresses critical gaps by evaluating LLMs in **real-world scenarios** through interaction with actual MCP servers, capturing real application challenges such as:
31
+
32
+ - 🎯 **Long-horizon reasoning** across multi-step tasks
33
+ - 🔧 **Large, unfamiliar tool spaces** with diverse MCP servers
34
+ - 🌍 **Real-world data sources** and live environments
35
+ - ⚡ **Dynamic evaluation** with time-sensitive ground truth
36
+
37
+
38
+ ## Table of Contents
39
+
40
+ - [What's New](#whats-new)
41
+ - [Architecture Overview](#architecture-overview)
42
+ - [Getting Started](#getting-started)
43
+ - [Prerequisites](#prerequisites)
44
+ - [Installation](#installation)
45
+ - [Quick Test](#quick-test)
46
+ - [Evaluating LLMs and Agents](#evaluating-llms-and-agents)
47
+ - [Prerequisites](#prerequisites-1)
48
+ - [Environment Configuration](#environment-configuration)
49
+ - [Benchmark Configuration](#benchmark-configuration)
50
+ - [Execution](#execution)
51
+ - [Save the running log](#save-the-running-log)
52
+ - [Save the benchmark result to a report](#save-the-benchmark-result-to-a-report)
53
+ - [Visualize the agent running information](#visualize-the-agent-running-information)
54
+ - [Creating Custom Benchmarks](#creating-custom-benchmarks)
55
+ - [Task definition](#task-definition)
56
+ - [Benchmark definition](#benchmark-definition)
57
+ - [Citation](#citation)
58
+
59
+ ## What's New
60
+
61
+ ### MCPMark Benchmark
62
+
63
+ **📊 Evaluate MCP Agents with MCPMark**
64
+
65
+ MCP-Universe now supports evaluating the **MCPMark** benchmark, enabling comprehensive testing and benchmarking of MCP agents. You can run MCPMark evaluations directly within the MCP-Universe framework to assess agent performance on MCP tasks.
66
+
67
+ **📚 Resources:**
68
+ - [How to run MCPMark](mcpuniverse/benchmark/configs/mcpmark/README.md#running-mcpmark-tasks)
69
+ - [Evaluation Scores](mcpuniverse/benchmark/configs/mcpmark/README.md#benchmark-results-alignment)
70
+
71
  ---
72
+
73
+ ### MCP+: Precision Context Management for MCP Agents
74
+
75
+ **🚀 Reduce LLM Token Costs by up to 75% Without Sacrificing Quality**
76
+
77
+ MCP tools often return large, verbose outputs that waste your LLM's context window and cost money. **MCP+** wraps your MCP clients with intelligent post-processing that extracts only the relevant information before it reaches your LLM.
78
+
79
+ #### ✨ Key Features
80
+
81
+ - **💰 Massive Cost Reduction**: 50-75% token savings on tool outputs
82
+ - **⚡ Zero Code Changes**: Drop-in replacement for standard MCP clients
83
+
84
+
85
+ **📚 [Learn More at mcp-plus.github.io →](https://mcp-plus.github.io)**
86
+
87
+ </div>
88
+
89
+ ---
90
+
91
+ ### Deep Research Agent: Wide & Deep (W&D) Research
92
+
93
+ **🔬 Scale Research Width with Parallel Tool Calls**
94
+
95
+ **Feb 11, 2026** — We introduce **Wide & Deep (W&D) research agents** that scale *width* by making more parallel tool calls per turn. This approach improves accuracy on BrowseComp, HLE, and GAIA benchmarks while reducing turns, API cost, and wall-clock time. Our W&D agent with GPT-5-medium reaches **62.2%** on BrowseComp, outperforming GPT-5-high deep research (54.9%).
96
+
97
+ **📚 Resources:**
98
+ - [Paper](https://arxiv.org/pdf/2602.07359)
99
+ - [Website](https://xqlin98.github.io/wide-deep-research-agent/)
100
+ - [Code](mcpuniverse/benchmark/configs/deepresearch/README.md)
101
+
102
+ ---
103
+
104
+ ## Architecture Overview
105
+
106
+ The MCPUniverse architecture consists of the following key components:
107
+
108
+ - **Agents** (`mcpuniverse/agent/`): Base implementations for different agent types
109
+ - **Workflows** (`mcpuniverse/workflows/`): Orchestration and coordination layer
110
+ - **MCP Servers** (`mcpuniverse/mcp/`): Protocol management and external service integration
111
+ - **LLM Integration** (`mcpuniverse/llm/`): Multi-provider language model support
112
+ - **Benchmarking** (`mcpuniverse/benchmark/`): Evaluation and testing framework
113
+ - **Dashboard** (`mcpuniverse/dashboard/`): Visualization and monitoring interface
114
+
115
+ The diagram below illustrates the high-level view:
116
+
117
+ ```
118
+ ┌─────────────────────────────────────────────────────────────────┐
119
+ │ Application Layer │
120
+ ├─────────────────────────────────────────────────────────────────┤
121
+ │ Dashboard │ Web API │ Python Lib │ Benchmarks │
122
+ │ (Gradio) │ (FastAPI) │ │ │
123
+ └─────────────┬─────────────────┬────────────────┬────────────────┘
124
+ │ │ │
125
+ ┌─────────────▼─────────────────▼────────────────▼────────────────┐
126
+ │ Orchestration Layer │
127
+ ├─────────────────────────────────────────────────────────────────┤
128
+ │ Workflows │ Benchmark Runner │
129
+ │ (Chain, Router, etc.) │ (Evaluation Engine) │
130
+ └─────────────┬─────────────────┬────────────────┬────────────────┘
131
+ │ │ │
132
+ ┌─────────────▼─────────────────▼────────────────▼────────────────┐
133
+ │ Agent Layer │
134
+ ├─────────────────────────────────────────────────────────────────┤
135
+ │ BasicAgent │ ReActAgent │ FunctionCall │ Other │
136
+ │ │ │ Agent │ Agents │
137
+ └─────────────┬─────────────────┬────────────────┬────────────────┘
138
+ │ │ │
139
+ ┌─────────────▼─────────────────▼────────────────▼────────────────┐
140
+ │ Foundation Layer │
141
+ ├─────────────────────────────────────────────────────────────────┤
142
+ │ MCP Manager │ LLM Manager │ Memory Systems │ Tracers │
143
+ │ (Servers & │ (Multi-Model │ (RAM, Redis) │ (Logging) │
144
+ │ Clients) │ Support) │ │ │
145
+ └─────────────────┴─────────────────┴─────────────────┴───────────┘
146
+ ```
147
+
148
+ More information can be found [here](https://github.com/SalesforceAIResearch/MCP-Universe/blob/main/docs).
149
+
150
+ ## Getting Started
151
+
152
+ We follow
153
+ the [feature branch workflow](https://www.atlassian.com/git/tutorials/comparing-workflows/feature-branch-workflow)
154
+ in this repo for its simplicity. To ensure code quality, [PyLint](https://pylint.readthedocs.io/en/latest/)
155
+ is integrated into our CI to enforce Python coding standards.
156
+
157
+ ### Prerequisites
158
+
159
+ * **Python**: Requires version 3.10 or higher.
160
+ * **Docker**: Used for running Dockerized MCP servers.
161
+ * **PostgreSQL** (optional): Used for database storage and persistence.
162
+ * **Redis** (optional): Used for caching and memory management.
163
+
164
+ ### Installation
165
+
166
+ 1. **Clone the repository**
167
+ ```bash
168
+ git clone https://github.com/SalesforceAIResearch/MCP-Universe.git
169
+ cd MCP-Universe
170
+ ```
171
+
172
+ 2. **Create and activate virtual environment**
173
+ ```bash
174
+ python3 -m venv venv
175
+ source venv/bin/activate
176
+ ```
177
+
178
+ 3. **Install dependencies**
179
+ ```bash
180
+ pip install -r requirements.txt
181
+ pip install -r dev-requirements.txt
182
+ ```
183
+
184
+ 4. **Platform-specific requirements**
185
+
186
+ **Linux:**
187
+ ```bash
188
+ sudo apt-get install libpq-dev
189
+ ```
190
+
191
+ **macOS:**
192
+ ```bash
193
+ brew install postgresql
194
+ ```
195
+
196
+ 5. **Configure pre-commit hooks**
197
+ ```bash
198
+ pre-commit install
199
+ ```
200
+
201
+ 6. **Environment configuration**
202
+ ```bash
203
+ cp .env.example .env
204
+ # Edit .env with your API keys and configuration
205
+ ```
206
+
207
+ ### Quick Test
208
+
209
+ To run benchmarks, you first need to set environment variables:
210
+
211
+ 1. Copy the `.env.example` file to a new file named `.env`.
212
+ 2. In the `.env` file, set the required API keys for various services used by the agents,
213
+ such as `OPENAI_API_KEY` and `GOOGLE_MAPS_API_KEY`.
214
+
215
+ To execute a benchmark programmatically:
216
+
217
+ ```python
218
+ from mcpuniverse.tracer.collectors import MemoryCollector # You can also use SQLiteCollector
219
+ from mcpuniverse.benchmark.runner import BenchmarkRunner
220
+
221
+ async def test():
222
+ trace_collector = MemoryCollector()
223
+ # Choose a benchmark config file under the folder "mcpuniverse/benchmark/configs"
224
+ benchmark = BenchmarkRunner("dummy/benchmark_1.yaml")
225
+ # Run the specified benchmark
226
+ results = await benchmark.run(trace_collector=trace_collector)
227
+ # Get traces
228
+ trace_id = results[0].task_trace_ids["dummy/tasks/weather_1.json"]
229
+ trace_records = trace_collector.get(trace_id)
230
+ ```
231
+
232
+ ## Evaluating LLMs and Agents
233
+
234
+ This section provides comprehensive instructions for evaluating LLMs and AI agents using the MCP-Universe benchmark suite. The framework supports evaluation across multiple domains including web search, location navigation, browser automation, financial analysis, repository management, and 3D design.
235
+
236
+ ### Prerequisites
237
+
238
+ Before running benchmark evaluations, ensure you have completed the [Getting Started](#getting-started) section and have the following:
239
+
240
+ - Python: Version 3.10 or higher
241
+ - Docker: Installed and available in your environment
242
+ - All required dependencies installed via `pip install -r requirements.txt`
243
+ - Active virtual environment
244
+ - Appropriate API access for the services you intend to evaluate
245
+
246
+ ### Environment Configuration
247
+
248
+ #### 1. Initial Setup
249
+
250
+ Copy the environment template and configure your API credentials:
251
+
252
+ ```bash
253
+ cp .env.example .env
254
+ ```
255
+
256
+ #### 2. API Keys and Configuration
257
+
258
+ Configure the following environment variables in your `.env` file. The required keys depend on which benchmark domains you plan to evaluate:
259
+
260
+ ##### Core LLM Providers
261
+
262
+ | Environment Variable | Provider | Description | Required For |
263
+ |---------------------|----------|-------------|--------------|
264
+ | `OPENAI_API_KEY` | OpenAI | API key for GPT models (gpt-5, etc.) | All domains |
265
+ | `ANTHROPIC_API_KEY` | Anthropic | API key for Claude models | All domains |
266
+ | `GEMINI_API_KEY` | Google | API key for Gemini models | All domains |
267
+
268
+ > **Note**: You only need to configure the API key for the LLM provider you intend to use in your evaluation.
269
+
270
+ ##### Domain-Specific Services
271
+
272
+ | Environment Variable | Service | Description | Setup Instructions |
273
+ |---------------------|---------|-------------|-------------------|
274
+ | `SERP_API_KEY` | SerpAPI | Web search API for search benchmark evaluation | [Get API key](https://serpapi.com/) |
275
+ | `GOOGLE_MAPS_API_KEY` | Google Maps | Geolocation and mapping services | [Setup Guide](https://console.cloud.google.com/google/maps-apis/credentials) |
276
+ | `GITHUB_PERSONAL_ACCESS_TOKEN` | GitHub | Personal access token for repository operations | [Token Setup](https://docs.github.com/en/authentication/keeping-your-account-and-data-secure/managing-your-personal-access-tokens) |
277
+ | `GITHUB_PERSONAL_ACCOUNT_NAME` | GitHub | Your GitHub username | N/A |
278
+ | `NOTION_API_KEY` | Notion | Integration token for Notion workspace access | [Integration Setup](https://developers.notion.com/docs/authorization#obtaining-a-token) |
279
+ | `NOTION_ROOT_PAGE` | Notion | Root page ID for your Notion workspace | See configuration example below |
280
+
281
+ ##### System Paths
282
+
283
+ | Environment Variable | Description | Example |
284
+ |---------------------|-------------|---------|
285
+ | `BLENDER_APP_PATH` | Full path to Blender executable (we used v4.4.0) | `/Applications/Blender.app/Contents/MacOS/Blender` |
286
+ | `MCPUniverse_DIR` | Absolute path to your MCP-Universe repository | `/Users/username/MCP-Universe` |
287
+
288
+ ##### Configuration Examples
289
+
290
+ **Notion Root Page ID:**
291
+ If your Notion page URL is:
292
+ ```
293
+ https://www.notion.so/your_workspace/MCP-Evaluation-1dd6d96e12345678901234567eaf9eff
294
+ ```
295
+ Set `NOTION_ROOT_PAGE=MCP-Evaluation-1dd6d96e12345678901234567eaf9eff`
296
+
297
+ **Blender Installation:**
298
+ 1. Download Blender v4.4.0 from [blender.org](https://www.blender.org/)
299
+ 2. Install our modified Blender MCP server following the [installation guide](docs/blender-setup.md)
300
+ 3. Set the path to the Blender executable
301
+
302
+ ##### ⚠️ Security Recommendations
303
+
304
+ > **🔒 IMPORTANT SECURITY NOTICE**
305
+ >
306
+ > Please read and follow these security guidelines carefully before running benchmarks:
307
+
308
+ - **🚨 GitHub Integration**: **CRITICAL** - We strongly recommend using a dedicated test GitHub account for benchmark evaluation. The AI agent will perform real operations on GitHub repositories, which could potentially modify or damage your personal repositories.
309
+
310
+ - **🔐 API Key Management**:
311
+ - Store API keys securely and never commit them to version control
312
+ - Use environment variables or secure key management systems
313
+ - Regularly rotate your API keys for enhanced security
314
+
315
+ - **🛡️ Access Permissions**:
316
+ - Grant minimal necessary permissions for each service integration
317
+ - Review and limit API key scopes to only required operations
318
+ - Monitor API usage and set appropriate rate limits
319
+
320
+ - **⚡ Blender Operations**: The 3D design benchmarks will execute Blender commands that may modify or create files on your system. Ensure you have adequate backups and run in an isolated environment if necessary.
321
+
322
+ ### Benchmark Configuration
323
+
324
+ #### Domain-Specific Configuration Files
325
+
326
+ Each benchmark domain has a dedicated YAML configuration file located in `mcpuniverse/benchmark/configs/test/`. To evaluate your LLM/agent, modify the appropriate configuration file:
327
+
328
+ | Domain | Configuration File | Description |
329
+ |--------|-------------------|-------------|
330
+ | Web Search | `web_search.yaml` | Search engine and information retrieval tasks |
331
+ | Location Navigation | `location_navigation.yaml` | Geographic and mapping-related queries |
332
+ | Browser Automation | `browser_automation.yaml` | Web interaction and automation scenarios |
333
+ | Financial Analysis | `financial_analysis.yaml` | Market data analysis and financial computations |
334
+ | Repository Management | `repository_management.yaml` | Git operations and code repository tasks |
335
+ | 3D Design | `3d_design.yaml` | Blender-based 3D modeling and design tasks |
336
+
337
+ #### LLM Model Configuration
338
+
339
+ In each configuration file, update the LLM specification to match your target model:
340
+
341
+ ```yaml
342
+ kind: llm
343
+ spec:
344
+ name: llm-1
345
+ type: openai # or anthropic, google, etc.
346
+ config:
347
+ model_name: gpt-4o # Replace with your target model
348
+ ```
349
+
350
+ ### Execution
351
+
352
+ #### Running Individual Benchmarks
353
+
354
+ Execute specific domain benchmarks using the following commands:
355
+
356
+ ```bash
357
+ # Set Python path and run individual benchmarks
358
+ export PYTHONPATH=.
359
+
360
+ # Location Navigation
361
+ python tests/benchmark/mcpuniverse/test_benchmark_location_navigation.py
362
+
363
+ # Browser Automation
364
+ python tests/benchmark/mcpuniverse/test_benchmark_browser_automation.py
365
+
366
+ # Financial Analysis
367
+ python tests/benchmark/mcpuniverse/test_benchmark_financial_analysis.py
368
+
369
+ # Repository Management
370
+ python tests/benchmark/mcpuniverse/test_benchmark_repository_management.py
371
+
372
+ # Web Search
373
+ python tests/benchmark/mcpuniverse/test_benchmark_web_search.py
374
+
375
+ # 3D Design
376
+ python tests/benchmark/mcpuniverse/test_benchmark_3d_design.py
377
+ ```
378
+
379
+ #### Batch Execution
380
+
381
+ For comprehensive evaluation across all domains:
382
+
383
+ ```bash
384
+ #!/bin/bash
385
+ export PYTHONPATH=.
386
+
387
+ domains=("location_navigation" "browser_automation" "financial_analysis"
388
+ "repository_management" "web_search" "3d_design")
389
+
390
+ for domain in "${domains[@]}"; do
391
+ echo "Running benchmark: $domain"
392
+ python "tests/benchmark/mcpuniverse/test_benchmark_${domain}.py"
393
+ echo "Completed: $domain"
394
+ done
395
+ ```
396
+
397
+ ### Save the running log
398
+
399
+ If you want to save the running log, you can pass the `trace_collector` to the benchmark run function:
400
+
401
+ ```python
402
+ from mcpuniverse.tracer.collectors import FileCollector
403
+
404
+ trace_collector = FileCollector(log_file="log/location_navigation.log")
405
+ benchmark_results = await benchmark.run(trace_collector=trace_collector)
406
+ ```
407
+
408
+ ### Save the benchmark result to a report
409
+
410
+ If you want to save a report of the benchmark result, you can use `BenchmarkReport` to dump a report:
411
+
412
+ ```python
413
+ from mcpuniverse.benchmark.report import BenchmarkReport
414
+
415
+ report = BenchmarkReport(benchmark, trace_collector=trace_collector)
416
+ report.dump()
417
+ ```
418
+
419
+ ### Visualize the agent running information
420
+
421
+ To run the benchmark with intermediate results and see real-time progress, pass `callbacks=get_vprint_callbacks()` to the run function:
422
+
423
+ ```python
424
+ from mcpuniverse.callbacks.handlers.vprint import get_vprint_callbacks
425
+
426
+ benchmark_results = await benchmark.run(
427
+ trace_collector=trace_collector,
428
+ callbacks=get_vprint_callbacks()
429
+ )
430
+ ```
431
+
432
+ This will print out the intermediate results as the benchmark runs.
433
+
434
+
435
+ For further details, refer to the in-code documentation or existing configuration samples in the repository.
436
+
437
+ ## Creating Custom Benchmarks
438
+
439
+ A benchmark is defined by three main configuration elements: the task definition,
440
+ agent/workflow definition, and the benchmark configuration itself. Below is an example
441
+ using a simple "weather forecasting" task.
442
+
443
+ ### Task definition
444
+
445
+ The task definition is provided in JSON format, for example:
446
+
447
+ ```json
448
+ {
449
+ "category": "general",
450
+ "question": "What's the weather in San Francisco now?",
451
+ "mcp_servers": [
452
+ {
453
+ "name": "weather"
454
+ }
455
+ ],
456
+ "output_format": {
457
+ "city": "<City>",
458
+ "weather": "<Weather forecast results>"
459
+ },
460
+ "evaluators": [
461
+ {
462
+ "func": "json -> get(city)",
463
+ "op": "=",
464
+ "value": "San Francisco"
465
+ }
466
+ ]
467
+ }
468
+ ```
469
+
470
+ Field descriptions:
471
+
472
+ 1. **category**: The task category, e.g., "general", "google-maps", etc. You can set any value for this property.
473
+ 2. **question**: The main question you want to ask in this task. This is treated as a user message.
474
+ 3. **mcp_servers**: A list of MCP servers that are supported in this framework.
475
+ 4. **output_format**: The desired output format of agent responses.
476
+ 5. **evaluators**: A list of tests to evaluate. For each test/evaluator, it has three attributes: "func" indicates
477
+ how to extract values from the agent response, "op" is the comparison operator, and "value" is the ground-truth
478
+ value.
479
+ It will evaluate **op(func(...), value, op_args...)**. "op" can be "=", "<", ">" or other customized operators.
480
+
481
+ In "evaluators", you need to write a rule ("func" attribute) showing how to extract values for testing. In the example
482
+ above, "json -> get(city)" will first do JSON decoding and then extract the value of key "city". There are several
483
+ predefined funcs in this repo:
484
+
485
+ 1. **json**: Perform JSON decoding.
486
+ 2. **get**: Get the value of a key.
487
+ 3. **len**: Get the length of a list.
488
+ 4. **foreach**: Do a FOR-EACH loop.
489
+
490
+ For example, let's define
491
+
492
+ ```python
493
+ data = {"x": [{"y": [1]}, {"y": [1, 1]}, {"y": [1, 2, 3, 4]}]}
494
+ ```
495
+
496
+ Then `get(x) -> foreach -> get(y) -> len` will do the following:
497
+
498
+ 1. Get the value of "x": `[{"y": [1]}, {"y": [1, 1]}, {"y": [1, 2, 3, 4]}]`.
499
+ 2. Do a foreach loop and get the value of "y": `[[1], [1, 1], [1, 2, 3, 4]]`.
500
+ 3. Get the length of each list: `[1, 2, 4]`.
501
+
502
+ If these predefined functions are not enough, you can implement custom ones.
503
+ For more details, please check
504
+ this [doc](https://github.com/SalesforceAIResearch/MCP-Universe/blob/main/docs/custom-evaluators-guide.md).
505
+
506
+ ### Benchmark definition
507
+
508
+ Define agent(s) and benchmark in a YAML file. Here’s a simple weather forecast benchmark:
509
+
510
+ ```yaml
511
+ kind: llm
512
+ spec:
513
+ name: llm-1
514
+ type: openai
515
+ config:
516
+ model_name: gpt-4o
517
+
518
+ ---
519
+ kind: agent
520
+ spec:
521
+ name: ReAct-agent
522
+ type: react
523
+ config:
524
+ llm: llm-1
525
+ instruction: You are an agent for weather forecasting.
526
+ servers:
527
+ - name: weather
528
+
529
+ ---
530
+ kind: benchmark
531
+ spec:
532
+ description: Test the agent for weather forecasting
533
+ agent: ReAct-agent
534
+ tasks:
535
+ - dummy/tasks/weather.json
536
+ ```
537
+
538
+ The benchmark definition mainly contains two parts: the agent definition and the benchmark configuration. The benchmark configuration is simple—you just need to specify the agent to use (by the defined agent name) and a list of tasks to evaluate. Each task entry is the task config file
539
+ path. It can be a full file path or a partial file path. If it is a partial file path (like "dummy/tasks/weather.json"),
540
+ it should be put in the
541
+ folder [mcpuniverse/benchmark/configs](https://github.com/SalesforceAIResearch/MCP-Universe/tree/main/mcpuniverse/benchmark/configs)
542
+ in this repo.
543
+
544
+ This framework offers a flexible way to define both simple agents (such as ReAct) and more complex, multi-step agent
545
+ workflows.
546
+
547
+ 1. **Specify LLMs:** Begin by declaring the large language models (LLMs) you want the agents to use. Each LLM component
548
+ must be assigned a unique name (e.g., `"llm-1"`). These names serve as identifiers that the framework uses to connect
549
+ the different components together.
550
+ 2. **Define an agent:** Next, define an agent by providing its name and selecting an agent class. Agent classes are
551
+ available in
552
+ the [mcpuniverse.agent](https://github.com/SalesforceAIResearch/MCP-Universe/tree/main/mcpuniverse/agent) package.
553
+ Commonly used classes include `"basic"`, `"function-call"`, and `"react"`. Within the agent specification (
554
+ `spec.config`), you must also indicate which LLM instance the agent should use by setting the `"llm"` field.
555
+ 3. **Create complex workflows:** Beyond simple agents, the framework supports the definition of sophisticated,
556
+ orchestrated workflows where multiple agents interact or collaborate to solve more complex tasks.
557
+
558
+ For example:
559
+
560
+ ```yaml
561
+ kind: llm
562
+ spec:
563
+ name: llm-1
564
+ type: openai
565
+ config:
566
+ model_name: gpt-4o
567
+
568
+ ---
569
+ kind: agent
570
+ spec:
571
+ name: basic-agent
572
+ type: basic
573
+ config:
574
+ llm: llm-1
575
+ instruction: Return the latitude and the longitude of a place.
576
+
577
+ ---
578
+ kind: agent
579
+ spec:
580
+ name: function-call-agent
581
+ type: function-call
582
+ config:
583
+ llm: llm-1
584
+ instruction: You are an agent for weather forecast. Please return the weather today at the given latitude and longitude.
585
+ servers:
586
+ - name: weather
587
+
588
+ ---
589
+ kind: workflow
590
+ spec:
591
+ name: orchestrator-workflow
592
+ type: orchestrator
593
+ config:
594
+ llm: llm-1
595
+ agents:
596
+ - basic-agent
597
+ - function-call-agent
598
+
599
+ ---
600
+ kind: benchmark
601
+ spec:
602
+ description: Test the agent for weather forecasting
603
+ agent: orchestrator-workflow
604
+ tasks:
605
+ - dummy/tasks/weather.json
606
+ ```
607
+
608
+ ## Citation
609
+
610
+ If you use MCP-Universe in your research, please cite our paper:
611
+
612
+ ```bibtex
613
+ @misc{mcpuniverse,
614
+ title={MCP-Universe: Benchmarking Large Language Models with Real-World Model Context Protocol Servers},
615
+ author={Ziyang Luo and Zhiqi Shen and Wenzhuo Yang and Zirui Zhao and Prathyusha Jwalapuram and Amrita Saha and Doyen Sahoo and Silvio Savarese and Caiming Xiong and Junnan Li},
616
+ year={2025},
617
+ eprint={2508.14704},
618
+ archivePrefix={arXiv},
619
+ primaryClass={cs.AI},
620
+ url={https://arxiv.org/abs/2508.14704},
621
+ }
622
+ ```
SECURITY.md ADDED
@@ -0,0 +1,7 @@
 
 
 
 
 
 
 
 
1
+ ## Security
2
+
3
+ Please report any security issue to [security@salesforce.com](mailto:security@salesforce.com)
4
+ as soon as it is discovered. This library limits its runtime dependencies in
5
+ order to reduce the total cost of ownership as much as can be, but all consumers
6
+ should remain vigilant and have their security stakeholders review all third-party
7
+ products (3PP) like this one and their dependencies.
blender_addon.py ADDED
@@ -0,0 +1,1458 @@
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
1
+ import bpy
2
+ import mathutils
3
+ import json
4
+ import threading
5
+ import socket
6
+ import time
7
+ import requests
8
+ import tempfile
9
+ import traceback
10
+ import os
11
+ import shutil
12
+ from bpy.props import StringProperty, IntProperty, BoolProperty, EnumProperty
13
+
14
+ bl_info = {
15
+ "name": "Blender MCP",
16
+ "author": "Based on ahujasid's BlenderMCP, Modified by the MCPWorld Team",
17
+ "version": (0, 3),
18
+ "blender": (3, 0, 0),
19
+ "location": "View3D > Sidebar > BlenderMCP",
20
+ "description": "Connect Blender to Agents via MCP",
21
+ "category": "Interface",
22
+ }
23
+
24
+ class BlenderMCPServer:
25
+ def __init__(self, host='localhost', port=9876):
26
+ self.host = host
27
+ self.port = port
28
+ self.running = False
29
+ self.socket = None
30
+ self.server_thread = None
31
+
32
+ def start(self):
33
+ if self.running:
34
+ print("Server is already running")
35
+ return True
36
+
37
+ self.running = True
38
+
39
+ try:
40
+ # Create socket
41
+ self.socket = socket.socket(socket.AF_INET, socket.SOCK_STREAM)
42
+ self.socket.setsockopt(socket.SOL_SOCKET, socket.SO_REUSEADDR, 1)
43
+ self.socket.bind((self.host, self.port))
44
+ self.socket.listen(1)
45
+
46
+ # Start server thread
47
+ self.server_thread = threading.Thread(target=self._server_loop)
48
+ self.server_thread.daemon = True
49
+ self.server_thread.start()
50
+
51
+ print(f"BlenderMCP server started on {self.host}:{self.port}")
52
+ return True
53
+ except Exception as e:
54
+ print(f"Failed to start server: {str(e)}")
55
+ self.stop()
56
+ return False
57
+
58
+ def stop(self):
59
+ self.running = False
60
+
61
+ # Close socket
62
+ if self.socket:
63
+ try:
64
+ self.socket.close()
65
+ except:
66
+ pass
67
+ self.socket = None
68
+
69
+ # Wait for thread to finish
70
+ if self.server_thread:
71
+ try:
72
+ if self.server_thread.is_alive():
73
+ self.server_thread.join(timeout=1.0)
74
+ except:
75
+ pass
76
+ self.server_thread = None
77
+
78
+ print("BlenderMCP server stopped")
79
+
80
+ def _server_loop(self):
81
+ """Main server loop in a separate thread"""
82
+ print("Server thread started")
83
+ self.socket.settimeout(1.0) # Timeout to allow for stopping
84
+
85
+ while self.running:
86
+ try:
87
+ # Accept new connection
88
+ try:
89
+ client, address = self.socket.accept()
90
+ print(f"Connected to client: {address}")
91
+
92
+ # Handle client in a separate thread
93
+ client_thread = threading.Thread(
94
+ target=self._handle_client,
95
+ args=(client,)
96
+ )
97
+ client_thread.daemon = True
98
+ client_thread.start()
99
+ except socket.timeout:
100
+ # Just check running condition
101
+ continue
102
+ except Exception as e:
103
+ print(f"Error accepting connection: {str(e)}")
104
+ time.sleep(0.5)
105
+ except Exception as e:
106
+ print(f"Error in server loop: {str(e)}")
107
+ if not self.running:
108
+ break
109
+ time.sleep(0.5)
110
+
111
+ print("Server thread stopped")
112
+
113
+ def _handle_client(self, client):
114
+ """Handle connected client"""
115
+ print("Client handler started")
116
+ client.settimeout(None) # No timeout
117
+ buffer = b''
118
+
119
+ try:
120
+ while self.running:
121
+ # Receive data
122
+ try:
123
+ data = client.recv(8192)
124
+ if not data:
125
+ print("Client disconnected")
126
+ break
127
+
128
+ buffer += data
129
+ try:
130
+ # Try to parse command
131
+ command = json.loads(buffer.decode('utf-8'))
132
+ buffer = b''
133
+
134
+ # Execute command in Blender's main thread
135
+ def execute_wrapper():
136
+ try:
137
+ response = self.execute_command(command)
138
+ response_json = json.dumps(response)
139
+ try:
140
+ client.sendall(response_json.encode('utf-8'))
141
+ except:
142
+ print("Failed to send response - client disconnected")
143
+ except Exception as e:
144
+ print(f"Error executing command: {str(e)}")
145
+ traceback.print_exc()
146
+ try:
147
+ error_response = {
148
+ "status": "error",
149
+ "message": str(e)
150
+ }
151
+ client.sendall(json.dumps(error_response).encode('utf-8'))
152
+ except:
153
+ pass
154
+ return None
155
+
156
+ # Schedule execution in main thread
157
+ bpy.app.timers.register(execute_wrapper, first_interval=0.0)
158
+ except json.JSONDecodeError:
159
+ # Incomplete data, wait for more
160
+ pass
161
+ except Exception as e:
162
+ print(f"Error receiving data: {str(e)}")
163
+ break
164
+ except Exception as e:
165
+ print(f"Error in client handler: {str(e)}")
166
+ finally:
167
+ try:
168
+ client.close()
169
+ except:
170
+ pass
171
+ print("Client handler stopped")
172
+
173
+ def execute_command(self, command):
174
+ """Execute a command in the main Blender thread"""
175
+ try:
176
+ cmd_type = command.get("type")
177
+ params = command.get("params", {})
178
+
179
+ # Ensure we're in the right context
180
+ if cmd_type in ["create_object", "modify_object", "delete_object"]:
181
+ override = bpy.context.copy()
182
+ override['area'] = [area for area in bpy.context.screen.areas if area.type == 'VIEW_3D'][0]
183
+ with bpy.context.temp_override(**override):
184
+ return self._execute_command_internal(command)
185
+ else:
186
+ return self._execute_command_internal(command)
187
+
188
+ except Exception as e:
189
+ print(f"Error executing command: {str(e)}")
190
+ traceback.print_exc()
191
+ return {"status": "error", "message": str(e)}
192
+
193
+ def _execute_command_internal(self, command):
194
+ """Internal command execution with proper context"""
195
+ cmd_type = command.get("type")
196
+ params = command.get("params", {})
197
+
198
+ # Add a handler for checking PolyHaven status
199
+ if cmd_type == "get_polyhaven_status":
200
+ return {"status": "success", "result": self.get_polyhaven_status()}
201
+
202
+ # Base handlers that are always available
203
+ handlers = {
204
+ "get_scene_info": self.get_scene_info,
205
+ "create_object": self.create_object,
206
+ "modify_object": self.modify_object,
207
+ "delete_object": self.delete_object,
208
+ "get_object_info": self.get_object_info,
209
+ "execute_code": self.execute_code,
210
+ "set_material": self.set_material,
211
+ "get_polyhaven_status": self.get_polyhaven_status,
212
+ }
213
+
214
+ # Add Polyhaven handlers - always enabled
215
+ polyhaven_handlers = {
216
+ "get_polyhaven_categories": self.get_polyhaven_categories,
217
+ "search_polyhaven_assets": self.search_polyhaven_assets,
218
+ "download_polyhaven_asset": self.download_polyhaven_asset,
219
+ "set_texture": self.set_texture,
220
+ "clear_scene_and_polyhaven_materials": self.clear_scene_and_polyhaven_materials,
221
+ }
222
+ handlers.update(polyhaven_handlers)
223
+
224
+ handler = handlers.get(cmd_type)
225
+ if handler:
226
+ try:
227
+ print(f"Executing handler for {cmd_type}")
228
+ result = handler(**params)
229
+ print(f"Handler execution complete")
230
+ return {"status": "success", "result": result}
231
+ except Exception as e:
232
+ print(f"Error in handler: {str(e)}")
233
+ traceback.print_exc()
234
+ return {"status": "error", "message": str(e)}
235
+ else:
236
+ return {"status": "error", "message": f"Unknown command type: {cmd_type}"}
237
+
238
+
239
+ def get_simple_info(self):
240
+ """Get basic Blender information"""
241
+ return {
242
+ "blender_version": ".".join(str(v) for v in bpy.app.version),
243
+ "scene_name": bpy.context.scene.name,
244
+ "object_count": len(bpy.context.scene.objects)
245
+ }
246
+
247
+ def get_scene_info(self):
248
+ """Get information about the current Blender scene"""
249
+ try:
250
+ print("Getting scene info...")
251
+ # Simplify the scene info to reduce data size
252
+ scene_info = {
253
+ "name": bpy.context.scene.name,
254
+ "object_count": len(bpy.context.scene.objects),
255
+ "objects": [],
256
+ "materials_count": len(bpy.data.materials),
257
+ }
258
+
259
+ # Collect minimal object information (limit to first 10 objects)
260
+ for i, obj in enumerate(bpy.context.scene.objects):
261
+ if i >= 10: # Reduced from 20 to 10
262
+ break
263
+
264
+ obj_info = {
265
+ "name": obj.name,
266
+ "type": obj.type,
267
+ # Only include basic location data
268
+ "location": [round(float(obj.location.x), 2),
269
+ round(float(obj.location.y), 2),
270
+ round(float(obj.location.z), 2)],
271
+ }
272
+ scene_info["objects"].append(obj_info)
273
+
274
+ print(f"Scene info collected: {len(scene_info['objects'])} objects")
275
+ return scene_info
276
+ except Exception as e:
277
+ print(f"Error in get_scene_info: {str(e)}")
278
+ traceback.print_exc()
279
+ return {"error": str(e)}
280
+
281
+ @staticmethod
282
+ def _get_aabb(obj):
283
+ """ Returns the world-space axis-aligned bounding box (AABB) of an object. """
284
+ if obj.type != 'MESH':
285
+ raise TypeError("Object must be a mesh")
286
+
287
+ # Get the bounding box corners in local space
288
+ local_bbox_corners = [mathutils.Vector(corner) for corner in obj.bound_box]
289
+
290
+ # Convert to world coordinates
291
+ world_bbox_corners = [obj.matrix_world @ corner for corner in local_bbox_corners]
292
+
293
+ # Compute axis-aligned min/max coordinates
294
+ min_corner = mathutils.Vector(map(min, zip(*world_bbox_corners)))
295
+ max_corner = mathutils.Vector(map(max, zip(*world_bbox_corners)))
296
+
297
+ return [
298
+ [*min_corner], [*max_corner]
299
+ ]
300
+
301
+ def create_object(self, type="CUBE", name=None, location=(0, 0, 0), rotation=(0, 0, 0), scale=(1, 1, 1),
302
+ align="WORLD", major_segments=48, minor_segments=12, mode="MAJOR_MINOR",
303
+ major_radius=1.0, minor_radius=0.25, abso_major_rad=1.25, abso_minor_rad=0.75, generate_uvs=True):
304
+ """Create a new object in the scene"""
305
+ try:
306
+ # Deselect all objects first
307
+ bpy.ops.object.select_all(action='DESELECT')
308
+
309
+ # Create the object based on type
310
+ if type == "CUBE":
311
+ bpy.ops.mesh.primitive_cube_add(location=location, rotation=rotation, scale=scale)
312
+ elif type == "SPHERE":
313
+ bpy.ops.mesh.primitive_uv_sphere_add(location=location, rotation=rotation, scale=scale)
314
+ elif type == "CYLINDER":
315
+ bpy.ops.mesh.primitive_cylinder_add(location=location, rotation=rotation, scale=scale)
316
+ elif type == "PLANE":
317
+ bpy.ops.mesh.primitive_plane_add(location=location, rotation=rotation, scale=scale)
318
+ elif type == "CONE":
319
+ bpy.ops.mesh.primitive_cone_add(location=location, rotation=rotation, scale=scale)
320
+ elif type == "TORUS":
321
+ bpy.ops.mesh.primitive_torus_add(
322
+ align=align,
323
+ location=location,
324
+ rotation=rotation,
325
+ major_segments=major_segments,
326
+ minor_segments=minor_segments,
327
+ mode=mode,
328
+ major_radius=major_radius,
329
+ minor_radius=minor_radius,
330
+ abso_major_rad=abso_major_rad,
331
+ abso_minor_rad=abso_minor_rad,
332
+ generate_uvs=generate_uvs
333
+ )
334
+ elif type == "EMPTY":
335
+ bpy.ops.object.empty_add(location=location, rotation=rotation, scale=scale)
336
+ elif type == "CAMERA":
337
+ bpy.ops.object.camera_add(location=location, rotation=rotation)
338
+ elif type == "LIGHT":
339
+ bpy.ops.object.light_add(type='POINT', location=location, rotation=rotation, scale=scale)
340
+ else:
341
+ raise ValueError(f"Unsupported object type: {type}")
342
+
343
+ # Force update the view layer
344
+ bpy.context.view_layer.update()
345
+
346
+ # Get the active object (which should be our newly created object)
347
+ obj = bpy.context.view_layer.objects.active
348
+
349
+ # If we don't have an active object, something went wrong
350
+ if obj is None:
351
+ raise RuntimeError("Failed to create object - no active object")
352
+
353
+ # Make sure it's selected
354
+ obj.select_set(True)
355
+
356
+ # Rename if name is provided
357
+ if name:
358
+ obj.name = name
359
+ if obj.data:
360
+ obj.data.name = name
361
+
362
+ # Return the object info
363
+ result = {
364
+ "name": obj.name,
365
+ "type": obj.type,
366
+ "location": [obj.location.x, obj.location.y, obj.location.z],
367
+ "rotation": [obj.rotation_euler.x, obj.rotation_euler.y, obj.rotation_euler.z],
368
+ "scale": [obj.scale.x, obj.scale.y, obj.scale.z],
369
+ }
370
+
371
+ if obj.type == "MESH":
372
+ bounding_box = self._get_aabb(obj)
373
+ result["world_bounding_box"] = bounding_box
374
+
375
+ return result
376
+ except Exception as e:
377
+ print(f"Error in create_object: {str(e)}")
378
+ traceback.print_exc()
379
+ return {"error": str(e)}
380
+
381
+ def modify_object(self, name, location=None, rotation=None, scale=None, visible=None):
382
+ """Modify an existing object in the scene"""
383
+ # Find the object by name
384
+ obj = bpy.data.objects.get(name)
385
+ if not obj:
386
+ raise ValueError(f"Object not found: {name}")
387
+
388
+ # Modify properties as requested
389
+ if location is not None:
390
+ obj.location = location
391
+
392
+ if rotation is not None:
393
+ obj.rotation_euler = rotation
394
+
395
+ if scale is not None:
396
+ obj.scale = scale
397
+
398
+ if visible is not None:
399
+ obj.hide_viewport = not visible
400
+ obj.hide_render = not visible
401
+
402
+ result = {
403
+ "name": obj.name,
404
+ "type": obj.type,
405
+ "location": [obj.location.x, obj.location.y, obj.location.z],
406
+ "rotation": [obj.rotation_euler.x, obj.rotation_euler.y, obj.rotation_euler.z],
407
+ "scale": [obj.scale.x, obj.scale.y, obj.scale.z],
408
+ "visible": obj.visible_get(),
409
+ }
410
+
411
+ if obj.type == "MESH":
412
+ bounding_box = self._get_aabb(obj)
413
+ result["world_bounding_box"] = bounding_box
414
+
415
+ return result
416
+
417
+ def delete_object(self, name):
418
+ """Delete an object from the scene"""
419
+ obj = bpy.data.objects.get(name)
420
+ if not obj:
421
+ raise ValueError(f"Object not found: {name}")
422
+
423
+ # Store the name to return
424
+ obj_name = obj.name
425
+
426
+ # Select and delete the object
427
+ if obj:
428
+ bpy.data.objects.remove(obj, do_unlink=True)
429
+
430
+ return {"deleted": obj_name}
431
+
432
+ def get_object_info(self, name):
433
+ """Get detailed information about a specific object"""
434
+ obj = bpy.data.objects.get(name)
435
+ if not obj:
436
+ raise ValueError(f"Object not found: {name}")
437
+
438
+ # Basic object info
439
+ obj_info = {
440
+ "name": obj.name,
441
+ "type": obj.type,
442
+ "location": [obj.location.x, obj.location.y, obj.location.z],
443
+ "rotation": [obj.rotation_euler.x, obj.rotation_euler.y, obj.rotation_euler.z],
444
+ "scale": [obj.scale.x, obj.scale.y, obj.scale.z],
445
+ "visible": obj.visible_get(),
446
+ "materials": [],
447
+ }
448
+
449
+ if obj.type == "MESH":
450
+ bounding_box = self._get_aabb(obj)
451
+ obj_info["world_bounding_box"] = bounding_box
452
+
453
+ # Add material slots
454
+ for slot in obj.material_slots:
455
+ if slot.material:
456
+ obj_info["materials"].append(slot.material.name)
457
+
458
+ # Add mesh data if applicable
459
+ if obj.type == 'MESH' and obj.data:
460
+ mesh = obj.data
461
+ obj_info["mesh"] = {
462
+ "vertices": len(mesh.vertices),
463
+ "edges": len(mesh.edges),
464
+ "polygons": len(mesh.polygons),
465
+ }
466
+
467
+ return obj_info
468
+
469
+ def execute_code(self, code):
470
+ """Execute arbitrary Blender Python code"""
471
+ # This is powerful but potentially dangerous - use with caution
472
+ try:
473
+ # Create a local namespace for execution
474
+ namespace = {"bpy": bpy}
475
+ exec(code, namespace)
476
+ return {"executed": True}
477
+ except Exception as e:
478
+ raise Exception(f"Code execution error: {str(e)}")
479
+
480
+ def set_material(self, object_name, material_name=None, create_if_missing=True, color=None):
481
+ """Set or create a material for an object"""
482
+ try:
483
+ # Get the object
484
+ obj = bpy.data.objects.get(object_name)
485
+ if not obj:
486
+ raise ValueError(f"Object not found: {object_name}")
487
+
488
+ # Make sure object can accept materials
489
+ if not hasattr(obj, 'data') or not hasattr(obj.data, 'materials'):
490
+ raise ValueError(f"Object {object_name} cannot accept materials")
491
+
492
+ # Create or get material
493
+ if material_name:
494
+ mat = bpy.data.materials.get(material_name)
495
+ if not mat and create_if_missing:
496
+ mat = bpy.data.materials.new(name=material_name)
497
+ print(f"Created new material: {material_name}")
498
+ else:
499
+ # Generate unique material name if none provided
500
+ mat_name = f"{object_name}_material"
501
+ mat = bpy.data.materials.get(mat_name)
502
+ if not mat:
503
+ mat = bpy.data.materials.new(name=mat_name)
504
+ material_name = mat_name
505
+ print(f"Using material: {mat_name}")
506
+
507
+ # Set up material nodes if needed
508
+ if mat:
509
+ if not mat.use_nodes:
510
+ mat.use_nodes = True
511
+
512
+ # Get or create Principled BSDF
513
+ principled = mat.node_tree.nodes.get('Principled BSDF')
514
+ if not principled:
515
+ principled = mat.node_tree.nodes.new('ShaderNodeBsdfPrincipled')
516
+ # Get or create Material Output
517
+ output = mat.node_tree.nodes.get('Material Output')
518
+ if not output:
519
+ output = mat.node_tree.nodes.new('ShaderNodeOutputMaterial')
520
+ # Link if not already linked
521
+ if not principled.outputs[0].links:
522
+ mat.node_tree.links.new(principled.outputs[0], output.inputs[0])
523
+
524
+ # Set color if provided
525
+ if color and len(color) >= 3:
526
+ principled.inputs['Base Color'].default_value = (
527
+ color[0],
528
+ color[1],
529
+ color[2],
530
+ 1.0 if len(color) < 4 else color[3]
531
+ )
532
+ print(f"Set material color to {color}")
533
+
534
+ # Assign material to object if not already assigned
535
+ if mat:
536
+ if not obj.data.materials:
537
+ obj.data.materials.append(mat)
538
+ else:
539
+ # Only modify first material slot
540
+ obj.data.materials[0] = mat
541
+
542
+ print(f"Assigned material {mat.name} to object {object_name}")
543
+
544
+ return {
545
+ "status": "success",
546
+ "object": object_name,
547
+ "material": mat.name,
548
+ "color": color if color else None
549
+ }
550
+ else:
551
+ raise ValueError(f"Failed to create or find material: {material_name}")
552
+
553
+ except Exception as e:
554
+ print(f"Error in set_material: {str(e)}")
555
+ traceback.print_exc()
556
+ return {
557
+ "status": "error",
558
+ "message": str(e),
559
+ "object": object_name,
560
+ "material": material_name if 'material_name' in locals() else None
561
+ }
562
+
563
+ def render_scene(self, output_path=None, resolution_x=None, resolution_y=None):
564
+ """Render the current scene"""
565
+ if resolution_x is not None:
566
+ bpy.context.scene.render.resolution_x = resolution_x
567
+
568
+ if resolution_y is not None:
569
+ bpy.context.scene.render.resolution_y = resolution_y
570
+
571
+ if output_path:
572
+ bpy.context.scene.render.filepath = output_path
573
+
574
+ # Render the scene
575
+ bpy.ops.render.render(write_still=bool(output_path))
576
+
577
+ return {
578
+ "rendered": True,
579
+ "output_path": output_path if output_path else "[not saved]",
580
+ "resolution": [bpy.context.scene.render.resolution_x, bpy.context.scene.render.resolution_y],
581
+ }
582
+
583
+ def get_polyhaven_categories(self, asset_type):
584
+ """Get categories for a specific asset type from Polyhaven"""
585
+ try:
586
+ if asset_type not in ["hdris", "textures", "models", "all"]:
587
+ return {"error": f"Invalid asset type: {asset_type}. Must be one of: hdris, textures, models, all"}
588
+
589
+ response = requests.get(f"https://api.polyhaven.com/categories/{asset_type}")
590
+ if response.status_code == 200:
591
+ return {"categories": response.json()}
592
+ else:
593
+ return {"error": f"API request failed with status code {response.status_code}"}
594
+ except Exception as e:
595
+ return {"error": str(e)}
596
+
597
+ def search_polyhaven_assets(self, asset_type=None, categories=None):
598
+ """Search for assets from Polyhaven with optional filtering"""
599
+ try:
600
+ url = "https://api.polyhaven.com/assets"
601
+ params = {}
602
+
603
+ if asset_type and asset_type != "all":
604
+ if asset_type not in ["hdris", "textures", "models"]:
605
+ return {"error": f"Invalid asset type: {asset_type}. Must be one of: hdris, textures, models, all"}
606
+ params["type"] = asset_type
607
+
608
+ if categories:
609
+ params["categories"] = categories
610
+
611
+ response = requests.get(url, params=params)
612
+ if response.status_code == 200:
613
+ # Limit the response size to avoid overwhelming Blender
614
+ assets = response.json()
615
+ # Return only the first 20 assets to keep response size manageable
616
+ limited_assets = {}
617
+ for i, (key, value) in enumerate(assets.items()):
618
+ if i >= 20: # Limit to 20 assets
619
+ break
620
+ limited_assets[key] = value
621
+
622
+ return {"assets": limited_assets, "total_count": len(assets), "returned_count": len(limited_assets)}
623
+ else:
624
+ return {"error": f"API request failed with status code {response.status_code}"}
625
+ except Exception as e:
626
+ return {"error": str(e)}
627
+
628
+ def download_polyhaven_asset(self, asset_id, asset_type, resolution="1k", file_format=None):
629
+ try:
630
+ # First get the files information
631
+ files_response = requests.get(f"https://api.polyhaven.com/files/{asset_id}")
632
+ if files_response.status_code != 200:
633
+ return {"error": f"Failed to get asset files: {files_response.status_code}"}
634
+
635
+ files_data = files_response.json()
636
+
637
+ # Handle different asset types
638
+ if asset_type == "hdris":
639
+ # For HDRIs, download the .hdr or .exr file
640
+ if not file_format:
641
+ file_format = "hdr" # Default format for HDRIs
642
+
643
+ if "hdri" in files_data and resolution in files_data["hdri"] and file_format in files_data["hdri"][resolution]:
644
+ file_info = files_data["hdri"][resolution][file_format]
645
+ file_url = file_info["url"]
646
+
647
+ # For HDRIs, we need to save to a temporary file first
648
+ # since Blender can't properly load HDR data directly from memory
649
+ with tempfile.NamedTemporaryFile(suffix=f".{file_format}", delete=False) as tmp_file:
650
+ # Download the file
651
+ response = requests.get(file_url)
652
+ if response.status_code != 200:
653
+ return {"error": f"Failed to download HDRI: {response.status_code}"}
654
+
655
+ tmp_file.write(response.content)
656
+ tmp_path = tmp_file.name
657
+
658
+ try:
659
+ # Create a new world if none exists
660
+ if not bpy.data.worlds:
661
+ bpy.data.worlds.new("World")
662
+
663
+ world = bpy.data.worlds[0]
664
+ world.use_nodes = True
665
+ node_tree = world.node_tree
666
+
667
+ # Clear existing nodes
668
+ for node in node_tree.nodes:
669
+ node_tree.nodes.remove(node)
670
+
671
+ # Create nodes
672
+ tex_coord = node_tree.nodes.new(type='ShaderNodeTexCoord')
673
+ tex_coord.location = (-800, 0)
674
+
675
+ mapping = node_tree.nodes.new(type='ShaderNodeMapping')
676
+ mapping.location = (-600, 0)
677
+
678
+ # Load the image from the temporary file
679
+ env_tex = node_tree.nodes.new(type='ShaderNodeTexEnvironment')
680
+ env_tex.location = (-400, 0)
681
+ env_tex.image = bpy.data.images.load(tmp_path)
682
+
683
+ # Use a color space that exists in all Blender versions
684
+ if file_format.lower() == 'exr':
685
+ # Try to use Linear color space for EXR files
686
+ try:
687
+ env_tex.image.colorspace_settings.name = 'Linear'
688
+ except:
689
+ # Fallback to Non-Color if Linear isn't available
690
+ env_tex.image.colorspace_settings.name = 'Non-Color'
691
+ else: # hdr
692
+ # For HDR files, try these options in order
693
+ for color_space in ['Linear', 'Linear Rec.709', 'Non-Color']:
694
+ try:
695
+ env_tex.image.colorspace_settings.name = color_space
696
+ break # Stop if we successfully set a color space
697
+ except:
698
+ continue
699
+
700
+ background = node_tree.nodes.new(type='ShaderNodeBackground')
701
+ background.location = (-200, 0)
702
+
703
+ output = node_tree.nodes.new(type='ShaderNodeOutputWorld')
704
+ output.location = (0, 0)
705
+
706
+ # Connect nodes
707
+ node_tree.links.new(tex_coord.outputs['Generated'], mapping.inputs['Vector'])
708
+ node_tree.links.new(mapping.outputs['Vector'], env_tex.inputs['Vector'])
709
+ node_tree.links.new(env_tex.outputs['Color'], background.inputs['Color'])
710
+ node_tree.links.new(background.outputs['Background'], output.inputs['Surface'])
711
+
712
+ # Set as active world
713
+ bpy.context.scene.world = world
714
+
715
+ # Clean up temporary file
716
+ try:
717
+ tempfile._cleanup() # This will clean up all temporary files
718
+ except:
719
+ pass
720
+
721
+ return {
722
+ "success": True,
723
+ "message": f"HDRI {asset_id} imported successfully",
724
+ "image_name": env_tex.image.name
725
+ }
726
+ except Exception as e:
727
+ return {"error": f"Failed to set up HDRI in Blender: {str(e)}"}
728
+ else:
729
+ return {"error": f"Requested resolution or format not available for this HDRI"}
730
+
731
+ elif asset_type == "textures":
732
+ if not file_format:
733
+ file_format = "jpg" # Default format for textures
734
+
735
+ downloaded_maps = {}
736
+
737
+ try:
738
+ for map_type in files_data:
739
+ if map_type not in ["blend", "gltf"]: # Skip non-texture files
740
+ if resolution in files_data[map_type] and file_format in files_data[map_type][resolution]:
741
+ file_info = files_data[map_type][resolution][file_format]
742
+ file_url = file_info["url"]
743
+
744
+ # Use NamedTemporaryFile like we do for HDRIs
745
+ with tempfile.NamedTemporaryFile(suffix=f".{file_format}", delete=False) as tmp_file:
746
+ # Download the file
747
+ response = requests.get(file_url)
748
+ if response.status_code == 200:
749
+ tmp_file.write(response.content)
750
+ tmp_path = tmp_file.name
751
+
752
+ # Load image from temporary file
753
+ image = bpy.data.images.load(tmp_path)
754
+ image.name = f"{asset_id}_{map_type}.{file_format}"
755
+
756
+ # Pack the image into .blend file
757
+ image.pack()
758
+
759
+ # Set color space based on map type
760
+ if map_type in ['color', 'diffuse', 'albedo']:
761
+ try:
762
+ image.colorspace_settings.name = 'sRGB'
763
+ except:
764
+ pass
765
+ else:
766
+ try:
767
+ image.colorspace_settings.name = 'Non-Color'
768
+ except:
769
+ pass
770
+
771
+ downloaded_maps[map_type] = image
772
+
773
+ # Clean up temporary file
774
+ try:
775
+ os.unlink(tmp_path)
776
+ except:
777
+ pass
778
+
779
+ if not downloaded_maps:
780
+ return {"error": f"No texture maps found for the requested resolution and format"}
781
+
782
+ # Create a new material with the downloaded textures
783
+ mat = bpy.data.materials.new(name=asset_id)
784
+ mat.use_nodes = True
785
+ nodes = mat.node_tree.nodes
786
+ links = mat.node_tree.links
787
+
788
+ # Clear default nodes
789
+ for node in nodes:
790
+ nodes.remove(node)
791
+
792
+ # Create output node
793
+ output = nodes.new(type='ShaderNodeOutputMaterial')
794
+ output.location = (300, 0)
795
+
796
+ # Create principled BSDF node
797
+ principled = nodes.new(type='ShaderNodeBsdfPrincipled')
798
+ principled.location = (0, 0)
799
+ links.new(principled.outputs[0], output.inputs[0])
800
+
801
+ # Add texture nodes based on available maps
802
+ tex_coord = nodes.new(type='ShaderNodeTexCoord')
803
+ tex_coord.location = (-800, 0)
804
+
805
+ mapping = nodes.new(type='ShaderNodeMapping')
806
+ mapping.location = (-600, 0)
807
+ mapping.vector_type = 'TEXTURE' # Changed from default 'POINT' to 'TEXTURE'
808
+ links.new(tex_coord.outputs['UV'], mapping.inputs['Vector'])
809
+
810
+ # Position offset for texture nodes
811
+ x_pos = -400
812
+ y_pos = 300
813
+
814
+ # Connect different texture maps
815
+ for map_type, image in downloaded_maps.items():
816
+ tex_node = nodes.new(type='ShaderNodeTexImage')
817
+ tex_node.location = (x_pos, y_pos)
818
+ tex_node.image = image
819
+
820
+ # Set color space based on map type
821
+ if map_type.lower() in ['color', 'diffuse', 'albedo']:
822
+ try:
823
+ tex_node.image.colorspace_settings.name = 'sRGB'
824
+ except:
825
+ pass # Use default if sRGB not available
826
+ else:
827
+ try:
828
+ tex_node.image.colorspace_settings.name = 'Non-Color'
829
+ except:
830
+ pass # Use default if Non-Color not available
831
+
832
+ links.new(mapping.outputs['Vector'], tex_node.inputs['Vector'])
833
+
834
+ # Connect to appropriate input on Principled BSDF
835
+ if map_type.lower() in ['color', 'diffuse', 'albedo']:
836
+ links.new(tex_node.outputs['Color'], principled.inputs['Base Color'])
837
+ elif map_type.lower() in ['roughness', 'rough']:
838
+ links.new(tex_node.outputs['Color'], principled.inputs['Roughness'])
839
+ elif map_type.lower() in ['metallic', 'metalness', 'metal']:
840
+ links.new(tex_node.outputs['Color'], principled.inputs['Metallic'])
841
+ elif map_type.lower() in ['normal', 'nor']:
842
+ # Add normal map node
843
+ normal_map = nodes.new(type='ShaderNodeNormalMap')
844
+ normal_map.location = (x_pos + 200, y_pos)
845
+ links.new(tex_node.outputs['Color'], normal_map.inputs['Color'])
846
+ links.new(normal_map.outputs['Normal'], principled.inputs['Normal'])
847
+ elif map_type in ['displacement', 'disp', 'height']:
848
+ # Add displacement node
849
+ disp_node = nodes.new(type='ShaderNodeDisplacement')
850
+ disp_node.location = (x_pos + 200, y_pos - 200)
851
+ links.new(tex_node.outputs['Color'], disp_node.inputs['Height'])
852
+ links.new(disp_node.outputs['Displacement'], output.inputs['Displacement'])
853
+
854
+ y_pos -= 250
855
+
856
+ return {
857
+ "success": True,
858
+ "message": f"Texture {asset_id} imported as material",
859
+ "material": mat.name,
860
+ "maps": list(downloaded_maps.keys())
861
+ }
862
+
863
+ except Exception as e:
864
+ return {"error": f"Failed to process textures: {str(e)}"}
865
+
866
+ elif asset_type == "models":
867
+ # For models, prefer glTF format if available
868
+ if not file_format:
869
+ file_format = "gltf" # Default format for models
870
+
871
+ if file_format in files_data and resolution in files_data[file_format]:
872
+ file_info = files_data[file_format][resolution][file_format]
873
+ file_url = file_info["url"]
874
+
875
+ # Create a temporary directory to store the model and its dependencies
876
+ temp_dir = tempfile.mkdtemp()
877
+ main_file_path = ""
878
+
879
+ try:
880
+ # Download the main model file
881
+ main_file_name = file_url.split("/")[-1]
882
+ main_file_path = os.path.join(temp_dir, main_file_name)
883
+
884
+ response = requests.get(file_url)
885
+ if response.status_code != 200:
886
+ return {"error": f"Failed to download model: {response.status_code}"}
887
+
888
+ with open(main_file_path, "wb") as f:
889
+ f.write(response.content)
890
+
891
+ # Check for included files and download them
892
+ if "include" in file_info and file_info["include"]:
893
+ for include_path, include_info in file_info["include"].items():
894
+ # Get the URL for the included file - this is the fix
895
+ include_url = include_info["url"]
896
+
897
+ # Create the directory structure for the included file
898
+ include_file_path = os.path.join(temp_dir, include_path)
899
+ os.makedirs(os.path.dirname(include_file_path), exist_ok=True)
900
+
901
+ # Download the included file
902
+ include_response = requests.get(include_url)
903
+ if include_response.status_code == 200:
904
+ with open(include_file_path, "wb") as f:
905
+ f.write(include_response.content)
906
+ else:
907
+ print(f"Failed to download included file: {include_path}")
908
+
909
+ # Import the model into Blender
910
+ if file_format == "gltf" or file_format == "glb":
911
+ bpy.ops.import_scene.gltf(filepath=main_file_path)
912
+ elif file_format == "fbx":
913
+ bpy.ops.import_scene.fbx(filepath=main_file_path)
914
+ elif file_format == "obj":
915
+ bpy.ops.import_scene.obj(filepath=main_file_path)
916
+ elif file_format == "blend":
917
+ # For blend files, we need to append or link
918
+ with bpy.data.libraries.load(main_file_path, link=False) as (data_from, data_to):
919
+ data_to.objects = data_from.objects
920
+
921
+ # Link the objects to the scene
922
+ for obj in data_to.objects:
923
+ if obj is not None:
924
+ bpy.context.collection.objects.link(obj)
925
+ else:
926
+ return {"error": f"Unsupported model format: {file_format}"}
927
+
928
+ # Get the names of imported objects
929
+ imported_objects = [obj.name for obj in bpy.context.selected_objects]
930
+
931
+ return {
932
+ "success": True,
933
+ "message": f"Model {asset_id} imported successfully",
934
+ "imported_objects": imported_objects
935
+ }
936
+ except Exception as e:
937
+ return {"error": f"Failed to import model: {str(e)}"}
938
+ finally:
939
+ # Clean up temporary directory
940
+ try:
941
+ shutil.rmtree(temp_dir)
942
+ except:
943
+ print(f"Failed to clean up temporary directory: {temp_dir}")
944
+ else:
945
+ return {"error": f"Requested format or resolution not available for this model"}
946
+
947
+ else:
948
+ return {"error": f"Unsupported asset type: {asset_type}"}
949
+
950
+ except Exception as e:
951
+ return {"error": f"Failed to download asset: {str(e)}"}
952
+
953
+ def set_texture(self, object_name, texture_id):
954
+ """Apply a previously downloaded Polyhaven texture to an object by creating a new material"""
955
+ try:
956
+ # Get the object
957
+ obj = bpy.data.objects.get(object_name)
958
+ if not obj:
959
+ return {"error": f"Object not found: {object_name}"}
960
+
961
+ # Make sure object can accept materials
962
+ if not hasattr(obj, 'data') or not hasattr(obj.data, 'materials'):
963
+ return {"error": f"Object {object_name} cannot accept materials"}
964
+
965
+ # Find all images related to this texture and ensure they're properly loaded
966
+ texture_images = {}
967
+ for img in bpy.data.images:
968
+ if img.name.startswith(texture_id + "_"):
969
+ # Extract the map type from the image name
970
+ map_type = img.name.split('_')[-1].split('.')[0]
971
+
972
+ # Force a reload of the image
973
+ img.reload()
974
+
975
+ # Ensure proper color space
976
+ if map_type.lower() in ['color', 'diffuse', 'albedo']:
977
+ try:
978
+ img.colorspace_settings.name = 'sRGB'
979
+ except:
980
+ pass
981
+ else:
982
+ try:
983
+ img.colorspace_settings.name = 'Non-Color'
984
+ except:
985
+ pass
986
+
987
+ # Ensure the image is packed
988
+ if not img.packed_file:
989
+ img.pack()
990
+
991
+ texture_images[map_type] = img
992
+ print(f"Loaded texture map: {map_type} - {img.name}")
993
+
994
+ # Debug info
995
+ print(f"Image size: {img.size[0]}x{img.size[1]}")
996
+ print(f"Color space: {img.colorspace_settings.name}")
997
+ print(f"File format: {img.file_format}")
998
+ print(f"Is packed: {bool(img.packed_file)}")
999
+
1000
+ if not texture_images:
1001
+ return {"error": f"No texture images found for: {texture_id}. Please download the texture first."}
1002
+
1003
+ # Create a new material
1004
+ new_mat_name = f"{texture_id}_material_{object_name}"
1005
+
1006
+ # Remove any existing material with this name to avoid conflicts
1007
+ existing_mat = bpy.data.materials.get(new_mat_name)
1008
+ if existing_mat:
1009
+ bpy.data.materials.remove(existing_mat)
1010
+
1011
+ new_mat = bpy.data.materials.new(name=new_mat_name)
1012
+ new_mat.use_nodes = True
1013
+
1014
+ # Set up the material nodes
1015
+ nodes = new_mat.node_tree.nodes
1016
+ links = new_mat.node_tree.links
1017
+
1018
+ # Clear default nodes
1019
+ nodes.clear()
1020
+
1021
+ # Create output node
1022
+ output = nodes.new(type='ShaderNodeOutputMaterial')
1023
+ output.location = (600, 0)
1024
+
1025
+ # Create principled BSDF node
1026
+ principled = nodes.new(type='ShaderNodeBsdfPrincipled')
1027
+ principled.location = (300, 0)
1028
+ links.new(principled.outputs[0], output.inputs[0])
1029
+
1030
+ # Add texture nodes based on available maps
1031
+ tex_coord = nodes.new(type='ShaderNodeTexCoord')
1032
+ tex_coord.location = (-800, 0)
1033
+
1034
+ mapping = nodes.new(type='ShaderNodeMapping')
1035
+ mapping.location = (-600, 0)
1036
+ mapping.vector_type = 'TEXTURE' # Changed from default 'POINT' to 'TEXTURE'
1037
+ links.new(tex_coord.outputs['UV'], mapping.inputs['Vector'])
1038
+
1039
+ # Position offset for texture nodes
1040
+ x_pos = -400
1041
+ y_pos = 300
1042
+
1043
+ # Connect different texture maps
1044
+ for map_type, image in texture_images.items():
1045
+ tex_node = nodes.new(type='ShaderNodeTexImage')
1046
+ tex_node.location = (x_pos, y_pos)
1047
+ tex_node.image = image
1048
+
1049
+ # Set color space based on map type
1050
+ if map_type.lower() in ['color', 'diffuse', 'albedo']:
1051
+ try:
1052
+ tex_node.image.colorspace_settings.name = 'sRGB'
1053
+ except:
1054
+ pass # Use default if sRGB not available
1055
+ else:
1056
+ try:
1057
+ tex_node.image.colorspace_settings.name = 'Non-Color'
1058
+ except:
1059
+ pass # Use default if Non-Color not available
1060
+
1061
+ links.new(mapping.outputs['Vector'], tex_node.inputs['Vector'])
1062
+
1063
+ # Connect to appropriate input on Principled BSDF
1064
+ if map_type.lower() in ['color', 'diffuse', 'albedo']:
1065
+ links.new(tex_node.outputs['Color'], principled.inputs['Base Color'])
1066
+ elif map_type.lower() in ['roughness', 'rough']:
1067
+ links.new(tex_node.outputs['Color'], principled.inputs['Roughness'])
1068
+ elif map_type.lower() in ['metallic', 'metalness', 'metal']:
1069
+ links.new(tex_node.outputs['Color'], principled.inputs['Metallic'])
1070
+ elif map_type.lower() in ['normal', 'nor', 'dx', 'gl']:
1071
+ # Add normal map node
1072
+ normal_map = nodes.new(type='ShaderNodeNormalMap')
1073
+ normal_map.location = (x_pos + 200, y_pos)
1074
+ links.new(tex_node.outputs['Color'], normal_map.inputs['Color'])
1075
+ links.new(normal_map.outputs['Normal'], principled.inputs['Normal'])
1076
+ elif map_type.lower() in ['displacement', 'disp', 'height']:
1077
+ # Add displacement node
1078
+ disp_node = nodes.new(type='ShaderNodeDisplacement')
1079
+ disp_node.location = (x_pos + 200, y_pos - 200)
1080
+ disp_node.inputs['Scale'].default_value = 0.1 # Reduce displacement strength
1081
+ links.new(tex_node.outputs['Color'], disp_node.inputs['Height'])
1082
+ links.new(disp_node.outputs['Displacement'], output.inputs['Displacement'])
1083
+
1084
+ y_pos -= 250
1085
+
1086
+ # Second pass: Connect nodes with proper handling for special cases
1087
+ texture_nodes = {}
1088
+
1089
+ # First find all texture nodes and store them by map type
1090
+ for node in nodes:
1091
+ if node.type == 'TEX_IMAGE' and node.image:
1092
+ for map_type, image in texture_images.items():
1093
+ if node.image == image:
1094
+ texture_nodes[map_type] = node
1095
+ break
1096
+
1097
+ # Now connect everything using the nodes instead of images
1098
+ # Handle base color (diffuse)
1099
+ for map_name in ['color', 'diffuse', 'albedo']:
1100
+ if map_name in texture_nodes:
1101
+ links.new(texture_nodes[map_name].outputs['Color'], principled.inputs['Base Color'])
1102
+ print(f"Connected {map_name} to Base Color")
1103
+ break
1104
+
1105
+ # Handle roughness
1106
+ for map_name in ['roughness', 'rough']:
1107
+ if map_name in texture_nodes:
1108
+ links.new(texture_nodes[map_name].outputs['Color'], principled.inputs['Roughness'])
1109
+ print(f"Connected {map_name} to Roughness")
1110
+ break
1111
+
1112
+ # Handle metallic
1113
+ for map_name in ['metallic', 'metalness', 'metal']:
1114
+ if map_name in texture_nodes:
1115
+ links.new(texture_nodes[map_name].outputs['Color'], principled.inputs['Metallic'])
1116
+ print(f"Connected {map_name} to Metallic")
1117
+ break
1118
+
1119
+ # Handle normal maps
1120
+ for map_name in ['gl', 'dx', 'nor']:
1121
+ if map_name in texture_nodes:
1122
+ normal_map_node = nodes.new(type='ShaderNodeNormalMap')
1123
+ normal_map_node.location = (100, 100)
1124
+ links.new(texture_nodes[map_name].outputs['Color'], normal_map_node.inputs['Color'])
1125
+ links.new(normal_map_node.outputs['Normal'], principled.inputs['Normal'])
1126
+ print(f"Connected {map_name} to Normal")
1127
+ break
1128
+
1129
+ # Handle displacement
1130
+ for map_name in ['displacement', 'disp', 'height']:
1131
+ if map_name in texture_nodes:
1132
+ disp_node = nodes.new(type='ShaderNodeDisplacement')
1133
+ disp_node.location = (300, -200)
1134
+ disp_node.inputs['Scale'].default_value = 0.1 # Reduce displacement strength
1135
+ links.new(texture_nodes[map_name].outputs['Color'], disp_node.inputs['Height'])
1136
+ links.new(disp_node.outputs['Displacement'], output.inputs['Displacement'])
1137
+ print(f"Connected {map_name} to Displacement")
1138
+ break
1139
+
1140
+ # Handle ARM texture (Ambient Occlusion, Roughness, Metallic)
1141
+ if 'arm' in texture_nodes:
1142
+ separate_rgb = nodes.new(type='ShaderNodeSeparateRGB')
1143
+ separate_rgb.location = (-200, -100)
1144
+ links.new(texture_nodes['arm'].outputs['Color'], separate_rgb.inputs['Image'])
1145
+
1146
+ # Connect Roughness (G) if no dedicated roughness map
1147
+ if not any(map_name in texture_nodes for map_name in ['roughness', 'rough']):
1148
+ links.new(separate_rgb.outputs['G'], principled.inputs['Roughness'])
1149
+ print("Connected ARM.G to Roughness")
1150
+
1151
+ # Connect Metallic (B) if no dedicated metallic map
1152
+ if not any(map_name in texture_nodes for map_name in ['metallic', 'metalness', 'metal']):
1153
+ links.new(separate_rgb.outputs['B'], principled.inputs['Metallic'])
1154
+ print("Connected ARM.B to Metallic")
1155
+
1156
+ # For AO (R channel), multiply with base color if we have one
1157
+ base_color_node = None
1158
+ for map_name in ['color', 'diffuse', 'albedo']:
1159
+ if map_name in texture_nodes:
1160
+ base_color_node = texture_nodes[map_name]
1161
+ break
1162
+
1163
+ if base_color_node:
1164
+ mix_node = nodes.new(type='ShaderNodeMixRGB')
1165
+ mix_node.location = (100, 200)
1166
+ mix_node.blend_type = 'MULTIPLY'
1167
+ mix_node.inputs['Fac'].default_value = 0.8 # 80% influence
1168
+
1169
+ # Disconnect direct connection to base color
1170
+ for link in base_color_node.outputs['Color'].links:
1171
+ if link.to_socket == principled.inputs['Base Color']:
1172
+ links.remove(link)
1173
+
1174
+ # Connect through the mix node
1175
+ links.new(base_color_node.outputs['Color'], mix_node.inputs[1])
1176
+ links.new(separate_rgb.outputs['R'], mix_node.inputs[2])
1177
+ links.new(mix_node.outputs['Color'], principled.inputs['Base Color'])
1178
+ print("Connected ARM.R to AO mix with Base Color")
1179
+
1180
+ # Handle AO (Ambient Occlusion) if separate
1181
+ if 'ao' in texture_nodes:
1182
+ base_color_node = None
1183
+ for map_name in ['color', 'diffuse', 'albedo']:
1184
+ if map_name in texture_nodes:
1185
+ base_color_node = texture_nodes[map_name]
1186
+ break
1187
+
1188
+ if base_color_node:
1189
+ mix_node = nodes.new(type='ShaderNodeMixRGB')
1190
+ mix_node.location = (100, 200)
1191
+ mix_node.blend_type = 'MULTIPLY'
1192
+ mix_node.inputs['Fac'].default_value = 0.8 # 80% influence
1193
+
1194
+ # Disconnect direct connection to base color
1195
+ for link in base_color_node.outputs['Color'].links:
1196
+ if link.to_socket == principled.inputs['Base Color']:
1197
+ links.remove(link)
1198
+
1199
+ # Connect through the mix node
1200
+ links.new(base_color_node.outputs['Color'], mix_node.inputs[1])
1201
+ links.new(texture_nodes['ao'].outputs['Color'], mix_node.inputs[2])
1202
+ links.new(mix_node.outputs['Color'], principled.inputs['Base Color'])
1203
+ print("Connected AO to mix with Base Color")
1204
+
1205
+ # CRITICAL: Make sure to clear all existing materials from the object
1206
+ while len(obj.data.materials) > 0:
1207
+ obj.data.materials.pop(index=0)
1208
+
1209
+ # Assign the new material to the object
1210
+ obj.data.materials.append(new_mat)
1211
+
1212
+ # CRITICAL: Make the object active and select it
1213
+ bpy.context.view_layer.objects.active = obj
1214
+ obj.select_set(True)
1215
+
1216
+ # CRITICAL: Force Blender to update the material
1217
+ bpy.context.view_layer.update()
1218
+
1219
+ # Get the list of texture maps
1220
+ texture_maps = list(texture_images.keys())
1221
+
1222
+ # Get info about texture nodes for debugging
1223
+ material_info = {
1224
+ "name": new_mat.name,
1225
+ "has_nodes": new_mat.use_nodes,
1226
+ "node_count": len(new_mat.node_tree.nodes),
1227
+ "texture_nodes": []
1228
+ }
1229
+
1230
+ for node in new_mat.node_tree.nodes:
1231
+ if node.type == 'TEX_IMAGE' and node.image:
1232
+ connections = []
1233
+ for output in node.outputs:
1234
+ for link in output.links:
1235
+ connections.append(f"{output.name} → {link.to_node.name}.{link.to_socket.name}")
1236
+
1237
+ material_info["texture_nodes"].append({
1238
+ "name": node.name,
1239
+ "image": node.image.name,
1240
+ "colorspace": node.image.colorspace_settings.name,
1241
+ "connections": connections
1242
+ })
1243
+
1244
+ return {
1245
+ "success": True,
1246
+ "message": f"Created new material and applied texture {texture_id} to {object_name}",
1247
+ "material": new_mat.name,
1248
+ "maps": texture_maps,
1249
+ "material_info": material_info
1250
+ }
1251
+
1252
+ except Exception as e:
1253
+ print(f"Error in set_texture: {str(e)}")
1254
+ traceback.print_exc()
1255
+ return {"error": f"Failed to apply texture: {str(e)}"}
1256
+
1257
+ def get_polyhaven_status(self):
1258
+ """Get the current status of PolyHaven integration"""
1259
+ # Poly Haven is now always enabled
1260
+ return {"enabled": True, "message": "PolyHaven integration is always enabled and ready to use."}
1261
+
1262
+ def clear_scene_and_polyhaven_materials(self):
1263
+ """Clear the entire scene and remove all Polyhaven materials and images"""
1264
+ try:
1265
+ cleared_items = {
1266
+ "objects": 0,
1267
+ "materials": 0,
1268
+ "images": 0,
1269
+ "meshes": 0,
1270
+ "lights": 0,
1271
+ "cameras": 0
1272
+ }
1273
+
1274
+ # Switch to object mode if not already
1275
+ if bpy.context.mode != 'OBJECT':
1276
+ bpy.ops.object.mode_set(mode='OBJECT')
1277
+
1278
+ # Delete all objects in the scene
1279
+ bpy.ops.object.select_all(action='SELECT')
1280
+ cleared_items["objects"] = len(bpy.context.selected_objects)
1281
+ bpy.ops.object.delete()
1282
+
1283
+ # Remove all materials (not just Polyhaven ones, since we're clearing everything)
1284
+ materials_to_remove = list(bpy.data.materials)
1285
+ for mat in materials_to_remove:
1286
+ cleared_items["materials"] += 1
1287
+ bpy.data.materials.remove(mat)
1288
+
1289
+ # Remove all images (including Polyhaven textures)
1290
+ images_to_remove = list(bpy.data.images)
1291
+ for img in images_to_remove:
1292
+ # Skip built-in images like Viewer Node
1293
+ if not img.name.startswith("Viewer Node"):
1294
+ cleared_items["images"] += 1
1295
+ bpy.data.images.remove(img)
1296
+
1297
+ # Remove all meshes
1298
+ meshes_to_remove = list(bpy.data.meshes)
1299
+ for mesh in meshes_to_remove:
1300
+ cleared_items["meshes"] += 1
1301
+ bpy.data.meshes.remove(mesh)
1302
+
1303
+ # Remove all lights
1304
+ lights_to_remove = list(bpy.data.lights)
1305
+ for light in lights_to_remove:
1306
+ cleared_items["lights"] += 1
1307
+ bpy.data.lights.remove(light)
1308
+
1309
+ # Remove all cameras
1310
+ cameras_to_remove = list(bpy.data.cameras)
1311
+ for camera in cameras_to_remove:
1312
+ cleared_items["cameras"] += 1
1313
+ bpy.data.cameras.remove(camera)
1314
+
1315
+ # Reset world to default
1316
+ if bpy.context.scene.world:
1317
+ world = bpy.context.scene.world
1318
+ if world.use_nodes:
1319
+ world.node_tree.nodes.clear()
1320
+ world.use_nodes = False
1321
+
1322
+ # Force update
1323
+ bpy.context.view_layer.update()
1324
+
1325
+ return {
1326
+ "success": True,
1327
+ "message": "Scene completely cleared and all Polyhaven materials removed",
1328
+ "cleared_items": cleared_items
1329
+ }
1330
+
1331
+ except Exception as e:
1332
+ print(f"Error in clear_scene_and_polyhaven_materials: {str(e)}")
1333
+ traceback.print_exc()
1334
+ return {"error": f"Failed to clear scene: {str(e)}"}
1335
+
1336
+ # Auto-start timer function
1337
+ def auto_start_server():
1338
+ """Timer function to automatically start the server"""
1339
+ try:
1340
+ scene = bpy.context.scene
1341
+
1342
+ # Always start the server if it's not running (removed the auto_start check)
1343
+ if not scene.blendermcp_server_running:
1344
+ # Create a new server instance if it doesn't exist
1345
+ if not hasattr(bpy.types, "blendermcp_server") or not bpy.types.blendermcp_server:
1346
+ bpy.types.blendermcp_server = BlenderMCPServer(port=scene.blendermcp_port)
1347
+
1348
+ # Try to start the server
1349
+ if bpy.types.blendermcp_server.start():
1350
+ scene.blendermcp_server_running = True
1351
+ print("BlenderMCP server auto-started successfully")
1352
+ # Return 30 seconds to check if server is still running
1353
+ return 10.0
1354
+ else:
1355
+ print("Failed to auto-start BlenderMCP server")
1356
+ # Retry in 5 seconds
1357
+ return 3.0
1358
+ else:
1359
+ # Server is running, check again in 30 seconds to ensure it stays running
1360
+ return 10.0
1361
+ except Exception as e:
1362
+ print(f"Error in auto-start: {str(e)}")
1363
+ # Retry in 5 seconds
1364
+ return 5.0
1365
+
1366
+ # Blender UI Panel
1367
+ class BLENDERMCP_PT_Panel(bpy.types.Panel):
1368
+ bl_label = "Blender MCP"
1369
+ bl_idname = "BLENDERMCP_PT_Panel"
1370
+ bl_space_type = 'VIEW_3D'
1371
+ bl_region_type = 'UI'
1372
+ bl_category = 'BlenderMCP'
1373
+
1374
+ def draw(self, context):
1375
+ layout = self.layout
1376
+ scene = context.scene
1377
+
1378
+ # Add a header message
1379
+ layout.label(text="MCP Server is always active", icon='INFO')
1380
+ layout.separator()
1381
+
1382
+ # Show port as read-only information
1383
+ row = layout.row()
1384
+ row.enabled = False # Make it read-only
1385
+ row.prop(scene, "blendermcp_port")
1386
+
1387
+ # Show server status (read-only, no control buttons)
1388
+ if scene.blendermcp_server_running:
1389
+ layout.label(text=f"Status: Running on port {scene.blendermcp_port}", icon='CHECKMARK')
1390
+ else:
1391
+ layout.label(text="Status: Starting...", icon='TIME')
1392
+
1393
+ layout.separator()
1394
+ layout.label(text="Server cannot be manually stopped", icon='LOCKED')
1395
+
1396
+ # Registration functions
1397
+ def register():
1398
+ # Removed blendermcp_auto_start since it's always enabled now
1399
+
1400
+ bpy.types.Scene.blendermcp_port = IntProperty(
1401
+ name="Port",
1402
+ description="Port for the BlenderMCP server (read-only)",
1403
+ default=9876,
1404
+ min=1024,
1405
+ max=65535
1406
+ )
1407
+
1408
+ bpy.types.Scene.blendermcp_server_running = bpy.props.BoolProperty(
1409
+ name="Server Running",
1410
+ default=False
1411
+ )
1412
+
1413
+ bpy.types.Scene.blendermcp_use_polyhaven = bpy.props.BoolProperty(
1414
+ name="Use Poly Haven",
1415
+ description="Enable Poly Haven asset integration",
1416
+ default=True # Always enabled by default
1417
+ )
1418
+
1419
+ bpy.utils.register_class(BLENDERMCP_PT_Panel)
1420
+ # Removed registration of start/stop operators
1421
+
1422
+ # Try to start server immediately
1423
+ try:
1424
+ scene = bpy.context.scene
1425
+ bpy.types.blendermcp_server = BlenderMCPServer(port=scene.blendermcp_port)
1426
+ if bpy.types.blendermcp_server.start():
1427
+ scene.blendermcp_server_running = True
1428
+ print("BlenderMCP server started immediately on registration")
1429
+ except Exception as e:
1430
+ print(f"Failed to start server immediately: {str(e)}")
1431
+
1432
+ # Schedule persistent auto-start timer
1433
+ bpy.app.timers.register(auto_start_server, first_interval=0.5)
1434
+
1435
+ print("BlenderMCP addon registered with permanent server")
1436
+
1437
+ def unregister():
1438
+ # Stop the server if it's running
1439
+ if hasattr(bpy.types, "blendermcp_server") and bpy.types.blendermcp_server:
1440
+ bpy.types.blendermcp_server.stop()
1441
+ del bpy.types.blendermcp_server
1442
+
1443
+ # Unregister the auto-start timer if it exists
1444
+ if bpy.app.timers.is_registered(auto_start_server):
1445
+ bpy.app.timers.unregister(auto_start_server)
1446
+
1447
+ bpy.utils.unregister_class(BLENDERMCP_PT_Panel)
1448
+ # Removed unregistration of start/stop operators
1449
+
1450
+ # Removed blendermcp_auto_start deletion
1451
+ del bpy.types.Scene.blendermcp_port
1452
+ del bpy.types.Scene.blendermcp_server_running
1453
+ del bpy.types.Scene.blendermcp_use_polyhaven
1454
+
1455
+ print("BlenderMCP addon unregistered")
1456
+
1457
+ if __name__ == "__main__":
1458
+ register()
console-errors-successful-load.log ADDED
@@ -0,0 +1,4 @@
 
 
 
 
 
1
+ Total messages: 2 (Errors: 2, Warnings: 0)
2
+
3
+ [ERROR] Failed to load resource: the server responded with a status of 403 () @ https://www.rwsentosa.com/en/reservations/attraction-selection?ThemeParkCode=ACW&VisitDate=2026-07-05:0
4
+ [ERROR] Failed to load resource: the server responded with a status of 403 () @ https://www.rwsentosa.com/favicon.ico:0
dev-requirements.txt ADDED
@@ -0,0 +1,5 @@
 
 
 
 
 
 
1
+ pytest
2
+ pylint
3
+ pre-commit
4
+ pytest_postgresql
5
+ pytest_asyncio
docs/adding-mcp-servers.md ADDED
@@ -0,0 +1,473 @@
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
1
+ # Adding MCP Servers to MCPUniverse
2
+
3
+ This guide explains how to add new Model Control Protocol (MCP) servers to the MCPUniverse framework. There are three main approaches: creating custom Python MCP servers, integrating existing third-party servers, and connecting to remote MCP servers.
4
+
5
+ ## Overview
6
+
7
+ MCPUniverse uses a centralized server configuration system that manages different types of MCP servers. All server configurations are stored in `mcpuniverse/mcp/configs/server_list.json`, which defines how to launch, connect to, and manage each server.
8
+
9
+ ## 1. Adding Custom Python MCP Servers
10
+
11
+ ### Step 1: Create Your Server Implementation
12
+
13
+ Create a new directory in `mcpuniverse/mcp/servers/` for your server:
14
+
15
+ ```bash
16
+ mkdir mcpuniverse/mcp/servers/my_custom_server
17
+ ```
18
+
19
+ Create the server implementation files:
20
+
21
+ **mcpuniverse/mcp/servers/my_custom_server/server.py:**
22
+ ```python
23
+ """
24
+ A custom MCP server implementation
25
+ """
26
+ import click
27
+ from typing import Any
28
+ from mcp.server.fastmcp import FastMCP
29
+ from mcpuniverse.common.logger import get_logger
30
+
31
+
32
+ def build_server(port: int) -> FastMCP:
33
+ """
34
+ Initialize the MCP server.
35
+
36
+ Args:
37
+ port: Port for SSE transport
38
+
39
+ Returns:
40
+ The configured MCP server
41
+ """
42
+ mcp = FastMCP("my-custom-server", port=port)
43
+ logger = get_logger("my-custom-server")
44
+
45
+ @mcp.tool()
46
+ async def my_custom_tool(param1: str, param2: int = 10) -> str:
47
+ """
48
+ Description of what this tool does.
49
+
50
+ Args:
51
+ param1: Description of parameter 1
52
+ param2: Description of parameter 2 (optional)
53
+
54
+ Returns:
55
+ Result description
56
+ """
57
+ logger.info(f"Executing custom tool with {param1} and {param2}")
58
+
59
+ # Your custom logic here
60
+ result = f"Processed {param1} with value {param2}"
61
+ return result
62
+
63
+ @mcp.tool()
64
+ async def another_tool(data: dict) -> dict:
65
+ """Another tool that processes dictionary data."""
66
+ return {"processed": True, "original": data}
67
+
68
+ @mcp.resource("custom://data/{resource_id}")
69
+ def get_custom_resource(resource_id: str) -> str:
70
+ """Get a custom resource by ID."""
71
+ return f"Resource data for {resource_id}"
72
+
73
+ return mcp
74
+
75
+
76
+ @click.command()
77
+ @click.option(
78
+ "--transport",
79
+ type=click.Choice(["stdio", "sse"]),
80
+ default="stdio",
81
+ help="Transport type"
82
+ )
83
+ @click.option("--port", default="8000", help="Port to listen on for SSE")
84
+ def main(transport: str, port: str):
85
+ """Start the MCP server."""
86
+ logger = get_logger("my-custom-server")
87
+ logger.info("Starting my custom MCP server")
88
+
89
+ mcp = build_server(int(port))
90
+ mcp.run(transport=transport.lower())
91
+
92
+
93
+ if __name__ == "__main__":
94
+ main()
95
+ ```
96
+
97
+ **mcpuniverse/mcp/servers/my_custom_server/__init__.py:**
98
+ ```python
99
+ """My Custom MCP Server"""
100
+ ```
101
+
102
+ **mcpuniverse/mcp/servers/my_custom_server/__main__.py:**
103
+ ```python
104
+ import sys
105
+ from .server import main
106
+
107
+ sys.exit(main())
108
+ ```
109
+
110
+ ### Step 2: Register Your Server
111
+
112
+ Add your server configuration to `mcpuniverse/mcp/configs/server_list.json`:
113
+
114
+ ```json
115
+ {
116
+ "my-custom-server": {
117
+ "stdio": {
118
+ "command": "python3",
119
+ "args": [
120
+ "-m", "mcpuniverse.mcp.servers.my_custom_server"
121
+ ]
122
+ },
123
+ "sse": {
124
+ "command": "python3",
125
+ "args": [
126
+ "-m", "mcpuniverse.mcp.servers.my_custom_server",
127
+ "--transport", "sse",
128
+ "--port", "{{PORT}}"
129
+ ]
130
+ },
131
+ "env": {
132
+ "CUSTOM_API_KEY": "{{CUSTOM_API_KEY}}",
133
+ "CUSTOM_CONFIG": "{{CUSTOM_CONFIG_PATH}}"
134
+ }
135
+ }
136
+ }
137
+ ```
138
+
139
+ ### Step 3: Create Tests
140
+
141
+ Create test files in `tests/mcp/servers/my_custom_server/`:
142
+
143
+ **tests/mcp/servers/my_custom_server/test_my_custom_server.py:**
144
+ ```python
145
+ import unittest
146
+ from mcpuniverse.mcp.servers.my_custom_server.server import build_server
147
+
148
+
149
+ class TestMyCustomServer(unittest.IsolatedAsyncioTestCase):
150
+
151
+ def setUp(self):
152
+ self.server = build_server(port=12345)
153
+
154
+ async def test_server_tools(self):
155
+ tools = await self.server.list_tools()
156
+ tool_names = [tool.name for tool in tools]
157
+
158
+ self.assertIn("my_custom_tool", tool_names)
159
+ self.assertIn("another_tool", tool_names)
160
+
161
+ async def test_my_custom_tool(self):
162
+ result = await self.server.call_tool("my_custom_tool", {
163
+ "param1": "test",
164
+ "param2": 42
165
+ })
166
+ self.assertIn("Processed test with value 42", str(result))
167
+
168
+
169
+ if __name__ == "__main__":
170
+ unittest.main()
171
+ ```
172
+
173
+ ### Step 4: Usage in Agents
174
+
175
+ Use your server in agent configurations:
176
+
177
+ **agent_config.yaml:**
178
+ ```yaml
179
+ name: "test-agent"
180
+ instruction: "An agent that uses my custom server"
181
+ servers:
182
+ - name: "my-custom-server"
183
+ - name: "weather" # Can combine with other servers
184
+ ```
185
+
186
+ ## 2. Adding Existing Third-Party MCP Servers
187
+
188
+ ### NPM/Node.js Packages
189
+
190
+ For servers published as NPM packages, add them directly to the configuration:
191
+
192
+ ```json
193
+ {
194
+ "third-party-server": {
195
+ "stdio": {
196
+ "command": "npx",
197
+ "args": [
198
+ "-y",
199
+ "package-name-from-npm"
200
+ ]
201
+ },
202
+ "env": {
203
+ "API_KEY": "{{THIRD_PARTY_API_KEY}}"
204
+ }
205
+ }
206
+ }
207
+ ```
208
+
209
+ **Example with actual third-party servers:**
210
+ ```json
211
+ {
212
+ "github": {
213
+ "stdio": {
214
+ "command": "npx",
215
+ "args": [
216
+ "-y",
217
+ "@modelcontextprotocol/server-github"
218
+ ]
219
+ },
220
+ "env": {
221
+ "GITHUB_PERSONAL_ACCESS_TOKEN": "{{GITHUB_PERSONAL_ACCESS_TOKEN}}"
222
+ }
223
+ },
224
+
225
+ "filesystem": {
226
+ "stdio": {
227
+ "command": "npx",
228
+ "args": [
229
+ "-y",
230
+ "@modelcontextprotocol/server-filesystem",
231
+ "{{FILESYSTEM_DIRECTORY}}"
232
+ ]
233
+ }
234
+ }
235
+ }
236
+ ```
237
+
238
+ ### Python Packages
239
+
240
+ For Python packages available via pip:
241
+
242
+ ```json
243
+ {
244
+ "python-third-party": {
245
+ "stdio": {
246
+ "command": "python3",
247
+ "args": [
248
+ "-m", "third_party_package_name"
249
+ ]
250
+ },
251
+ "env": {
252
+ "PACKAGE_CONFIG": "{{PACKAGE_CONFIG_PATH}}"
253
+ }
254
+ }
255
+ }
256
+ ```
257
+
258
+ **Example:**
259
+ ```json
260
+ {
261
+ "calculator": {
262
+ "stdio": {
263
+ "command": "python3",
264
+ "args": [
265
+ "-m", "mcp_server_calculator"
266
+ ]
267
+ }
268
+ },
269
+
270
+ "fetch": {
271
+ "stdio": {
272
+ "command": "python3",
273
+ "args": [
274
+ "-m", "mcp_server_fetch",
275
+ "--ignore-robots-txt"
276
+ ]
277
+ }
278
+ }
279
+ }
280
+ ```
281
+
282
+ ### Binary Executables
283
+
284
+ For servers distributed as binaries:
285
+
286
+ ```json
287
+ {
288
+ "binary-server": {
289
+ "stdio": {
290
+ "command": "/path/to/binary",
291
+ "args": [
292
+ "--config", "{{CONFIG_PATH}}",
293
+ "--mode", "stdio"
294
+ ]
295
+ },
296
+ "sse": {
297
+ "command": "/path/to/binary",
298
+ "args": [
299
+ "--config", "{{CONFIG_PATH}}",
300
+ "--mode", "sse",
301
+ "--port", "{{PORT}}"
302
+ ]
303
+ }
304
+ }
305
+ }
306
+ ```
307
+
308
+ ## 3. Adding Remote MCP Servers
309
+
310
+ Using MCP remote proxy:
311
+
312
+ ```json
313
+ {
314
+ "proxied-remote": {
315
+ "stdio": {
316
+ "command": "npx",
317
+ "args": [
318
+ "mcp-remote",
319
+ "https://remote-mcp-server.com/sse"
320
+ ]
321
+ }
322
+ }
323
+ }
324
+ ```
325
+
326
+ ## Environment Variables and Configuration
327
+
328
+ ### Setting Environment Variables
329
+
330
+ Create a `.env` file in your project root:
331
+
332
+ ```bash
333
+ # Third-party API keys
334
+ GITHUB_PERSONAL_ACCESS_TOKEN=your_github_token_here
335
+ GOOGLE_MAPS_API_KEY=your_google_maps_key
336
+ SERP_API_KEY=your_serp_api_key
337
+
338
+ # Custom server configurations
339
+ CUSTOM_API_KEY=your_custom_api_key
340
+ FILESYSTEM_DIRECTORY=/path/to/allowed/directory
341
+
342
+ # Remote server authentication
343
+ REMOTE_API_TOKEN=your_remote_token
344
+ ```
345
+
346
+ ### Template Variables
347
+
348
+ The server configuration supports template variables that are replaced at runtime:
349
+
350
+ - `{{PORT}}`: Automatically assigned port for SSE transport
351
+ - Any environment variable in `{{VARIABLE_NAME}}` format
352
+
353
+ ## Usage Examples
354
+
355
+ ### Basic Server Usage
356
+
357
+ ```python
358
+ from mcpuniverse.mcp.manager import MCPManager
359
+ from mcpuniverse.agent.manager import AgentManager
360
+ from mcpuniverse.llm.manager import ModelManager
361
+
362
+ # Initialize components
363
+ mcp_manager = MCPManager()
364
+ llm = ModelManager().build_model(name="openai")
365
+ agent_manager = AgentManager()
366
+
367
+ # Create agent with your custom server
368
+ agent = agent_manager.build_agent(
369
+ class_name="function_call",
370
+ mcp_manager=mcp_manager,
371
+ llm=llm,
372
+ config={
373
+ "name": "test-agent",
374
+ "instruction": "Use custom tools to solve problems",
375
+ "servers": [
376
+ {"name": "my-custom-server"},
377
+ {"name": "weather"}
378
+ ]
379
+ }
380
+ )
381
+
382
+ # Use the agent
383
+ await agent.initialize()
384
+ response = await agent.execute("Use my custom tool with some data")
385
+ await agent.cleanup()
386
+ ```
387
+
388
+ ### Programmatic Server Management
389
+
390
+ ```python
391
+ from mcpuniverse.mcp.manager import MCPManager
392
+
393
+ manager = MCPManager()
394
+
395
+ # Build client for specific server
396
+ client = await manager.build_client("my-custom-server", transport="stdio")
397
+ # List available tools
398
+ tools = await client.list_tools()
399
+ print(f"Available tools: {tools}")
400
+ # Execute a tool
401
+ result = await client.execute_tool("my_custom_tool", {
402
+ "param1": "hello",
403
+ "param2": 123
404
+ })
405
+ print(f"Tool result: {result}")
406
+ await client.cleanup()
407
+ ```
408
+
409
+ ### Dynamic Server Registration
410
+
411
+ Register servers dynamically at runtime:
412
+
413
+ ```python
414
+ manager = MCPManager()
415
+
416
+ # Add server configuration dynamically
417
+ new_server_config = {
418
+ "stdio": {
419
+ "command": "python3",
420
+ "args": ["-m", "my.dynamic.server"]
421
+ }
422
+ }
423
+
424
+ manager.add_server_config("dynamic-server", new_server_config)
425
+ ```
426
+
427
+ ## Troubleshooting
428
+
429
+ ### Common Issues
430
+
431
+ 1. **Server Not Found**: Ensure the server name in `server_list.json` matches what you use in agent configs
432
+ 2. **Command Not Found**: Verify the command and arguments are correct and the package is installed
433
+ 3. **Environment Variables**: Check that all required environment variables are set in `.env` or your environment
434
+ 4**Permission Issues**: Verify file permissions for binary executables
435
+
436
+ ### Debugging
437
+
438
+ Enable debug logging:
439
+
440
+ ```python
441
+ import logging
442
+ logging.basicConfig(level=logging.DEBUG)
443
+
444
+ from mcpuniverse.mcp.manager import MCPManager
445
+ manager = MCPManager()
446
+ ```
447
+
448
+ Test server connectivity:
449
+
450
+ ```python
451
+ # Test if server can be reached
452
+ client = await manager.build_client("server-name")
453
+ try:
454
+ tools = await client.list_tools()
455
+ print(f"Success! Tools: {tools}")
456
+ except Exception as e:
457
+ print(f"Failed to connect: {e}")
458
+ finally:
459
+ await client.cleanup()
460
+ ```
461
+
462
+ ## Best Practices
463
+
464
+ 1. **Documentation**: Document your tools with clear descriptions and parameter types
465
+ 2. **Error Handling**: Implement proper error handling in your server tools
466
+ 3. **Testing**: Write comprehensive tests for your server functionality
467
+ 4. **Security**: Never hardcode API keys; always use environment variables
468
+ 5. **Performance**: Consider async operations for I/O bound tasks
469
+ 6. **Logging**: Use structured logging for debugging and monitoring
470
+ 7. **Versioning**: Version your custom servers and maintain backward compatibility
471
+ 8. **Resource Management**: Properly clean up resources in your server implementation
472
+
473
+ This guide provides a comprehensive overview of adding MCP servers to MCPUniverse. Choose the approach that best fits your use case, and refer to the existing server implementations in the codebase for additional examples and patterns.
docs/blender-setup.md ADDED
@@ -0,0 +1,135 @@
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
1
+ # Blender MCP Server Setup Guide
2
+
3
+ This guide will help you set up the Blender MCP server integration within the MCP-Universe project.
4
+
5
+ ## Acknowledgments
6
+
7
+ This Blender MCP integration is built upon the excellent work by [@ahujasid](https://github.com/ahujasid) and the [blender-mcp project](https://github.com/ahujasid/blender-mcp). We extend our gratitude for their contribution to the MCP ecosystem.
8
+
9
+ ## Overview
10
+
11
+ The Blender MCP server enables LLMs to directly interact with and control Blender through the Model Context Protocol. This integration allows for:
12
+
13
+ - **AI-assisted 3D modeling**: Create and modify 3D objects through natural language
14
+ - **Scene manipulation**: Control lighting, cameras, and materials
15
+ - **Asset integration**: Download and use assets from Poly Haven
16
+ - **Code execution**: Run arbitrary Python code in Blender
17
+ - **Real-time collaboration**: Two-way communication between LLMs and Blender
18
+
19
+ ## Prerequisites
20
+
21
+ Before starting, ensure you have:
22
+
23
+ - **Blender 3.0 or newer** installed
24
+ - **Python 3.10 or newer**
25
+ - **uv package manager** (required for MCP server management)
26
+
27
+ ### Installing uv Package Manager
28
+
29
+ **For macOS:**
30
+ ```bash
31
+ brew install uv
32
+ ```
33
+
34
+ **For Windows:**
35
+ ```powershell
36
+ powershell -c "irm https://astral.sh/uv/install.ps1 | iex"
37
+ ```
38
+ Then add to PATH:
39
+ ```cmd
40
+ set Path=C:\Users\%USERNAME%\.local\bin;%Path%
41
+ ```
42
+
43
+ **For Linux/Other platforms:**
44
+ See the [official uv installation guide](https://docs.astral.sh/uv/getting-started/installation/)
45
+
46
+ ⚠️ **Important**: Do not proceed without installing uv first!
47
+
48
+ ## Installation Steps
49
+
50
+ ### Step 1: Download the Blender Addon
51
+
52
+ Download the Blender addon from our project:
53
+ - **Addon file**: [Download from Google Drive](https://drive.google.com/file/d/1o3SCsPQUXKf7y3anuyvhwvN5Zd1xHcR0/view?usp=drive_link)
54
+
55
+ ### Step 2: Install the Blender Addon
56
+
57
+ 1. Open **Blender**
58
+ 2. Navigate to **Edit > Preferences > Add-ons**
59
+ 3. Click **"Install..."** button
60
+ 4. Select the downloaded `addon.py` file
61
+ 5. **Enable the addon** by checking the box next to "Interface: Blender MCP"
62
+ 6. The addon should now appear in your Blender interface
63
+
64
+ ### Step 3: Configure MCP Server
65
+
66
+ #### For Claude Desktop Integration
67
+
68
+ 1. Open Claude Desktop
69
+ 2. Go to **Claude > Settings > Developer > Edit Config**
70
+ 3. Edit `claude_desktop_config.json` and add the following configuration:
71
+
72
+ ```json
73
+ {
74
+ "mcpServers": {
75
+ "blender": {
76
+ "command": "uvx",
77
+ "args": [
78
+ "blender-mcp"
79
+ ]
80
+ }
81
+ }
82
+ }
83
+ ```
84
+
85
+ #### For Cursor Integration
86
+
87
+ **Option 1: Global MCP Server**
88
+ 1. Go to **Settings > MCP**
89
+ 2. Click **"Add new global MCP server"**
90
+ 3. Use the following configuration:
91
+
92
+ ```json
93
+ {
94
+ "mcpServers": {
95
+ "blender": {
96
+ "command": "uvx",
97
+ "args": [
98
+ "blender-mcp"
99
+ ]
100
+ }
101
+ }
102
+ }
103
+ ```
104
+
105
+ **Option 2: Project-specific Server**
106
+ 1. Create `.cursor/mcp.json` in your project root
107
+ 2. Add the same configuration as above
108
+
109
+ **For Windows Cursor users:**
110
+ Use this configuration instead:
111
+ ```json
112
+ {
113
+ "mcpServers": {
114
+ "blender": {
115
+ "command": "cmd",
116
+ "args": [
117
+ "/c",
118
+ "uvx",
119
+ "blender-mcp"
120
+ ]
121
+ }
122
+ }
123
+ }
124
+ ```
125
+
126
+ ⚠️ **Note**: Only run one instance of the MCP server (either Claude Desktop OR Cursor), not both simultaneously.
127
+
128
+ ## Security Considerations
129
+
130
+ ⚠️ **Important Security Notes:**
131
+
132
+ - The Blender MCP server can execute arbitrary Python code in Blender
133
+ - Always save your work before using code execution features
134
+ - Use with caution in production environments
135
+ - Consider the implications of automated asset downloads
docs/configuration-guide.md ADDED
@@ -0,0 +1,541 @@
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
1
+ # MCPUniverse Configuration Guide
2
+
3
+ This guide provides comprehensive documentation for writing agent, workflow, and benchmark configurations in MCPUniverse.
4
+
5
+ ## Table of Contents
6
+
7
+ 1. [Overview](#overview)
8
+ 2. [LLM Configuration](#llm-configuration)
9
+ 3. [Agent Configuration](#agent-configuration)
10
+ 4. [Workflow Configuration](#workflow-configuration)
11
+ 5. [Benchmark Configuration](#benchmark-configuration)
12
+ 6. [Task Definition](#task-definition)
13
+ 7. [Complete Examples](#complete-examples)
14
+ 8. [Best Practices](#best-practices)
15
+
16
+ ## Overview
17
+
18
+ Component configurations use YAML documents separated by `---` delimiters. Each document represents a component (LLM, agent, workflow, or benchmark) and follows this structure:
19
+
20
+ ```yaml
21
+ kind: <component_type>
22
+ spec:
23
+ name: <component_name>
24
+ type: <implementation_type>
25
+ config:
26
+ # Component-specific configuration
27
+ ```
28
+
29
+ ## LLM Configuration
30
+
31
+ The LLM configuration defines language models used by agents and workflows.
32
+
33
+ ### Basic Structure
34
+
35
+ ```yaml
36
+ kind: llm
37
+ spec:
38
+ name: <unique_name>
39
+ type: <provider_type>
40
+ config:
41
+ model_name: <model_identifier>
42
+ # Additional provider-specific settings
43
+ ```
44
+
45
+ ### Example Providers
46
+
47
+ #### OpenAI
48
+ ```yaml
49
+ kind: llm
50
+ spec:
51
+ name: gpt-4o-llm
52
+ type: openai
53
+ config:
54
+ model_name: gpt-4o
55
+ temperature: 1.0
56
+ max_completion_tokens: 2000
57
+ ```
58
+
59
+ #### Claude (Anthropic)
60
+ ```yaml
61
+ kind: llm
62
+ spec:
63
+ name: claude-llm
64
+ type: claude
65
+ config:
66
+ model_name: claude-3-5-sonnet-20241022
67
+ temperature: 0.1
68
+ max_completion_tokens: 4000
69
+ ```
70
+
71
+ #### Google Gemini
72
+ ```yaml
73
+ kind: llm
74
+ spec:
75
+ name: gemini-llm
76
+ type: gemini
77
+ config:
78
+ model_name: gemini-2.0-flash
79
+ temperature: 0.5
80
+ ```
81
+
82
+ ## Agent Configuration
83
+
84
+ Agents are the core execution units that interact with LLMs and MCP servers.
85
+
86
+ ### Basic Structure
87
+
88
+ ```yaml
89
+ kind: agent
90
+ spec:
91
+ name: <unique_name>
92
+ type: <agent_type>
93
+ config:
94
+ llm: <llm_name (if required)>
95
+ instruction: <system_instruction>
96
+ # Agent-specific configuration
97
+ ```
98
+
99
+ ### Agent Types
100
+
101
+ #### Basic Agent
102
+ Simple LLM calling agent without tool use.
103
+
104
+ ```yaml
105
+ kind: agent
106
+ spec:
107
+ name: basic-agent
108
+ type: basic
109
+ config:
110
+ llm: gpt-4o-llm
111
+ instruction: You are a helpful assistant that provides information.
112
+ ```
113
+
114
+ #### Function Call Agent
115
+ Stateless agent that makes function calls to MCP servers.
116
+
117
+ ```yaml
118
+ kind: agent
119
+ spec:
120
+ name: function-call-agent
121
+ type: function-call
122
+ config:
123
+ llm: gpt-4o-llm
124
+ instruction: You are an agent that can call functions to help users.
125
+ servers:
126
+ - name: weather
127
+ - name: google-maps
128
+ ```
129
+
130
+ #### ReAct Agent
131
+ Reasoning and acting agent that follows the ReAct pattern.
132
+
133
+ ```yaml
134
+ kind: agent
135
+ spec:
136
+ name: react-agent
137
+ type: react
138
+ config:
139
+ llm: gpt-4o-llm
140
+ instruction: You are a ReAct agent that reasons and acts.
141
+ max_iterations: 10
142
+ servers:
143
+ - name: weather
144
+ - name: google-search
145
+ ```
146
+
147
+ #### Reflection Agent
148
+ Agent that uses reflection for improved reasoning.
149
+
150
+ ```yaml
151
+ kind: agent
152
+ spec:
153
+ name: reflection-agent
154
+ type: reflection
155
+ config:
156
+ llm: gpt-4o-llm
157
+ instruction: You are a reflection agent that improves through self-reflection.
158
+ max_iterations: 5
159
+ servers:
160
+ - name: weather
161
+ ```
162
+
163
+ ### Agent Configuration Parameters
164
+
165
+ | Parameter | Type | Description | Default |
166
+ |-----------|------|---------------------------------------------------|---------|
167
+ | `llm` | string | LLM component name | - |
168
+ | `instruction` | string | System instruction/prompt | - |
169
+ | `servers` | list | MCP servers to connect to | [] |
170
+ | `system_prompt` | string | Custom system prompt template path | - |
171
+ | `max_iterations` | int | Max reasoning iterations (ReAct/Reflection) | 5 |
172
+ | `summarize_tool_response` | bool | Summarize tool responses using LLM (ReAct) | false |
173
+ | `use_llm_tool_api` | string | Enable LLM's native tool calling API ("yes"/"no") | "no" |
174
+ | `mcp_gateway_url` | string | MCP gateway server URL for remote tool access | "" |
175
+
176
+ ### Advanced Agent Configuration
177
+
178
+ #### LLM Tool API Integration
179
+
180
+ The `use_llm_tool_api` parameter enables integration with the LLM provider's native tool calling API, allowing for more efficient tool execution,
181
+ and the `mcp_gateway_url` parameter enables remote MCP server access through a gateway:
182
+
183
+ ```yaml
184
+ kind: agent
185
+ spec:
186
+ name: remote-agent
187
+ type: basic
188
+ config:
189
+ llm: gpt-4.1-llm
190
+ instruction: You are an agent that uses remote MCP servers.
191
+ use_llm_tool_api: "yes"
192
+ mcp_gateway_url: "https://your-gateway.example.com"
193
+ servers:
194
+ - name: weather
195
+ - name: google-search
196
+ ```
197
+
198
+ **Key Features:**
199
+ - **Remote Access**: Connect to MCP servers hosted on remote machines
200
+ - **SSE Transport**: Uses Server-Sent Events for communication
201
+ - **Gateway URL**: Points to the MCP gateway server endpoint
202
+ - **Integration**: Works with `use_llm_tool_api: "yes"` for optimal performance
203
+
204
+ ## Workflow Configuration
205
+
206
+ Workflows orchestrate multiple agents to complete complex tasks.
207
+
208
+ ### Basic Structure
209
+
210
+ ```yaml
211
+ kind: workflow
212
+ spec:
213
+ name: <unique_name>
214
+ type: <workflow_type>
215
+ config:
216
+ # Workflow-specific configuration
217
+ ```
218
+
219
+ ### Workflow Types
220
+
221
+ #### Orchestrator Workflow
222
+ Coordinates multiple agents based on planning.
223
+
224
+ ```yaml
225
+ kind: workflow
226
+ spec:
227
+ name: orchestrator-workflow
228
+ type: orchestrator
229
+ config:
230
+ llm: gpt-4o-llm
231
+ agents:
232
+ - basic-agent
233
+ - function-call-agent
234
+ plan_type: "full" # or "iterative"
235
+ max_iterations: 10
236
+ ```
237
+
238
+ #### Evaluator-Optimizer Workflow
239
+ Executes the Evaluator-Optimizer workflow for iterative response improvement.
240
+
241
+ ```yaml
242
+ kind: workflow
243
+ spec:
244
+ name: evaluator-optimizer-workflow
245
+ type: evaluator-optimizer
246
+ config:
247
+ optimizer: agent1
248
+ evaluator: agent2
249
+ max_iterations: 5
250
+ ```
251
+
252
+ #### Chain Workflow
253
+ Sequential execution of agents.
254
+
255
+ ```yaml
256
+ kind: workflow
257
+ spec:
258
+ name: chain-workflow
259
+ type: chain
260
+ config:
261
+ agents:
262
+ - agent1
263
+ - agent2
264
+ - agent3
265
+ ```
266
+
267
+ #### Parallelization Workflow
268
+ Executes multiple agents in parallel.
269
+
270
+ ```yaml
271
+ kind: workflow
272
+ spec:
273
+ name: parallel-workflow
274
+ type: parallelization
275
+ config:
276
+ agents:
277
+ - agent1
278
+ - agent2
279
+ aggregator: agent3
280
+ ```
281
+
282
+ #### Router Workflow
283
+ Routes tasks to appropriate agents based on criteria.
284
+
285
+ ```yaml
286
+ kind: workflow
287
+ spec:
288
+ name: router-workflow
289
+ type: router
290
+ config:
291
+ llm: gpt-4o-llm
292
+ agents:
293
+ - weather-agent
294
+ - maps-agent
295
+ - search-agent
296
+ ```
297
+
298
+ ## Benchmark Configuration
299
+
300
+ Benchmarks define evaluation scenarios for agents and workflows.
301
+
302
+ ### Basic Structure
303
+
304
+ ```yaml
305
+ kind: benchmark
306
+ spec:
307
+ description: <benchmark_description>
308
+ agent: <agent_or_workflow_name>
309
+ tasks:
310
+ - <task_file_path>
311
+ - <task_file_path>
312
+ ```
313
+
314
+ ### Example
315
+
316
+ ```yaml
317
+ kind: benchmark
318
+ spec:
319
+ description: Weather forecasting benchmark
320
+ agent: weather-agent
321
+ tasks:
322
+ - dummy/tasks/weather_1.json
323
+ - dummy/tasks/weather_2.json
324
+ ```
325
+
326
+ ### Benchmark Parameters
327
+
328
+ | Parameter | Type | Description | Required |
329
+ |-----------|------|-------------|----------|
330
+ | `description` | string | Human-readable description | Yes |
331
+ | `agent` | string | Target agent or workflow name | Yes |
332
+ | `tasks` | list | List of task file paths | Yes |
333
+
334
+ ### Task File Paths
335
+
336
+ Task paths can be:
337
+ - **Relative**: `dummy/tasks/weather.json` (relative to `mcpuniverse/benchmark/configs/`)
338
+ - **Absolute**: `/full/path/to/task.json`
339
+
340
+ ## Task Definition
341
+
342
+ Tasks are defined in JSON format and specify the evaluation criteria.
343
+
344
+ ### Basic Structure
345
+
346
+ ```json
347
+ {
348
+ "category": "task_category",
349
+ "question": "The task question or instruction",
350
+ "mcp_servers": [
351
+ {
352
+ "name": "server_name"
353
+ }
354
+ ],
355
+ "output_format": {
356
+ "field1": "expected_format",
357
+ "field2": "expected_format"
358
+ },
359
+ "evaluators": [
360
+ {
361
+ "func": "evaluation_function",
362
+ "op": "comparison_operator",
363
+ "value": "expected_value"
364
+ }
365
+ ]
366
+ }
367
+ ```
368
+
369
+ ### Task Components
370
+
371
+ #### MCP Servers
372
+ Specify required servers (optional):
373
+
374
+ ```json
375
+ "mcp_servers": [
376
+ {
377
+ "name": "weather"
378
+ },
379
+ {
380
+ "name": "google-maps",
381
+ }
382
+ ]
383
+ ```
384
+
385
+ #### Output Format
386
+ Define expected response structure:
387
+
388
+ ```json
389
+ "output_format": {
390
+ "city": "<City Name>",
391
+ "weather": "<Weather Description>",
392
+ "temperature": "<Temperature in Celsius>",
393
+ "forecast": [
394
+ {
395
+ "day": "<Day>",
396
+ "condition": "<Condition>"
397
+ }
398
+ ]
399
+ }
400
+ ```
401
+
402
+ #### Evaluators
403
+ Define evaluation criteria using function chains:
404
+
405
+ ```json
406
+ "evaluators": [
407
+ {
408
+ "func": "json -> get(city)",
409
+ "op": "=",
410
+ "value": "San Francisco"
411
+ },
412
+ {
413
+ "func": "json -> get(forecast) -> len",
414
+ "op": ">",
415
+ "value": 3
416
+ },
417
+ {
418
+ "func": "json -> get(forecast) -> foreach -> get(day)",
419
+ "op": "contains",
420
+ "value": "Monday"
421
+ }
422
+ ]
423
+ ```
424
+
425
+ #### Built-in Evaluation Functions
426
+
427
+ | Function | Description | Example |
428
+ |----------|-------------|---------|
429
+ | `json` | Parse JSON response | `json` |
430
+ | `get(key)` | Extract field value | `get(city)` |
431
+ | `len` | Get array/string length | `len` |
432
+ | `foreach` | Iterate over array | `foreach` |
433
+ | `contains` | Check if value exists | - |
434
+
435
+ #### Built-in Comparison Operators
436
+
437
+ | Operator | Description | Example |
438
+ |------------|-------------------------------|-----------------------|
439
+ | `=` | Exact equality | `"value": "expected"` |
440
+ | `>` | Greater than | `"value": 5` |
441
+ | `<` | Less than | `"value": 10` |
442
+ | `>=` | Greater than or equal | `"value": 0` |
443
+ | `<=` | Less than or equal | `"value": 100` |
444
+ | `in` | Check if a value is in a list | `"value": "list"` |
445
+ | `contains` | Contains substring/element | `"value": "keyword"` |
446
+
447
+ ## Complete Examples
448
+
449
+ ### Simple Weather Agent
450
+
451
+ ```yaml
452
+ kind: llm
453
+ spec:
454
+ name: gpt-4o-llm
455
+ type: openai
456
+ config:
457
+ model_name: gpt-4o
458
+ temperature: 0.1
459
+
460
+ ---
461
+ kind: agent
462
+ spec:
463
+ name: weather-agent
464
+ type: react
465
+ config:
466
+ llm: gpt-4o-llm
467
+ instruction: You are a weather forecasting agent.
468
+ max_iterations: 5
469
+ servers:
470
+ - name: weather
471
+
472
+ ---
473
+ kind: benchmark
474
+ spec:
475
+ description: Weather forecasting evaluation
476
+ agent: weather-agent
477
+ tasks:
478
+ - dummy/tasks/weather_1.json
479
+ - dummy/tasks/weather_2.json
480
+ ```
481
+
482
+ ### Multi-Agent Workflow
483
+
484
+ ```yaml
485
+ kind: llm
486
+ spec:
487
+ name: planning-llm
488
+ type: openai
489
+ config:
490
+ model_name: gpt-4o-mini
491
+
492
+ ---
493
+ kind: llm
494
+ spec:
495
+ name: execution-llm
496
+ type: openai
497
+ config:
498
+ model_name: gpt-4o
499
+
500
+ ---
501
+ kind: agent
502
+ spec:
503
+ name: location-agent
504
+ type: basic
505
+ config:
506
+ llm: execution-llm
507
+ instruction: Extract location information from user queries.
508
+
509
+ ---
510
+ kind: agent
511
+ spec:
512
+ name: weather-agent
513
+ type: function-call
514
+ config:
515
+ llm: execution-llm
516
+ instruction: Get weather information for specified locations.
517
+ servers:
518
+ - name: weather
519
+
520
+ ---
521
+ kind: workflow
522
+ spec:
523
+ name: travel-planner
524
+ type: orchestrator
525
+ config:
526
+ llm: planning-llm
527
+ agents:
528
+ - location-agent
529
+ - weather-agent
530
+
531
+ ---
532
+ kind: benchmark
533
+ spec:
534
+ description: Travel planning with weather consideration
535
+ agent: travel-planner
536
+ tasks:
537
+ - test/travel/travel_task_0001.json
538
+ - test/travel/travel_task_0002.json
539
+ ```
540
+
541
+ This guide provides the foundation for creating effective MCPUniverse configurations. For additional examples, refer to the `mcpuniverse/benchmark/configs/` directory in the repository.
docs/custom-agent-guide.md ADDED
@@ -0,0 +1,297 @@
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
1
+ # Custom Agent Implementation Guide
2
+
3
+ This guide explains how to implement custom agents in the MCPUniverse framework, building upon the existing agent architecture to create specialized AI agents.
4
+
5
+ ## Overview
6
+
7
+ MCPUniverse provides a flexible agent framework that allows you to create custom agents with different reasoning patterns, tool usage, and behaviors. The framework is built around the Model Control Protocol (MCP) and supports various LLM providers.
8
+
9
+ ## Architecture
10
+
11
+ ### Core Components
12
+
13
+ 1. **BaseAgent**: Abstract base class that all agents inherit from
14
+ 2. **BaseAgentConfig**: Configuration class for agent parameters
15
+ 3. **AgentResponse**: Standardized response format
16
+ 4. **MCPManager**: Manages connections to MCP servers
17
+ 5. **Tracer**: Handles execution tracing and debugging
18
+
19
+ ### Agent Types
20
+
21
+ The framework includes several built-in agent types:
22
+
23
+ - **BasicAgent**: Simple LLM interaction agent
24
+ - **ReAct**: Reasoning and Acting agent implementation
25
+ - **FunctionCallAgent**: Uses LLM native tool calling APIs
26
+ - **ReflectionAgent**: Self-reflective agent with memory
27
+
28
+ ## Creating a Custom Agent
29
+
30
+ ### Step 1: Define Your Agent Configuration
31
+
32
+ Create a configuration class that extends `BaseAgentConfig`:
33
+
34
+ ```python
35
+ from dataclasses import dataclass
36
+ from mcpuniverse.agent.base import BaseAgentConfig
37
+
38
+ @dataclass
39
+ class MyCustomAgentConfig(BaseAgentConfig):
40
+ """Configuration for your custom agent."""
41
+
42
+ # Add custom configuration parameters
43
+ max_retries: int = 3
44
+ temperature: float = 0.7
45
+ enable_memory: bool = True
46
+ custom_prompt_path: str = "custom_prompt.j2"
47
+
48
+ # You can override default values
49
+ system_prompt: str = "path/to/your/custom_system_prompt.j2"
50
+ max_iterations: int = 10
51
+ ```
52
+
53
+ ### Step 2: Implement Your Custom Agent Class
54
+
55
+ Create your agent class by inheriting from `BaseAgent`:
56
+
57
+ ```python
58
+ from typing import Optional, Union, Dict, List
59
+ from mcpuniverse.agent.base import BaseAgent
60
+ from mcpuniverse.agent.types import AgentResponse
61
+ from mcpuniverse.mcp.manager import MCPManager
62
+ from mcpuniverse.llm.base import BaseLLM
63
+ from mcpuniverse.tracer import Tracer
64
+ from mcpuniverse.agent.utils import build_system_prompt
65
+
66
+ class MyCustomAgent(BaseAgent):
67
+ """A custom agent implementation."""
68
+
69
+ # Required class attributes
70
+ config_class = MyCustomAgentConfig
71
+ alias = ["custom", "my-agent"] # Alternative names for agent registration
72
+
73
+ def __init__(
74
+ self,
75
+ mcp_manager: Optional[MCPManager] = None,
76
+ llm: BaseLLM = None,
77
+ config: Optional[Union[Dict, str]] = None,
78
+ **kwargs
79
+ ):
80
+ """Initialize your custom agent."""
81
+ super().__init__(mcp_manager=mcp_manager, llm=llm, config=config)
82
+
83
+ # Initialize any custom attributes
84
+ self._custom_memory = []
85
+ self._retry_count = 0
86
+
87
+ async def _initialize(self):
88
+ """Optional: Initialize custom resources."""
89
+ # This method is called after MCP clients are set up
90
+ # Add any custom initialization logic here
91
+ pass
92
+
93
+ async def _execute(
94
+ self,
95
+ message: Union[str, List[str]],
96
+ **kwargs
97
+ ) -> AgentResponse:
98
+ """Main execution method - implement your agent logic here."""
99
+
100
+ # Get tracer for debugging
101
+ tracer = kwargs.get("tracer", Tracer())
102
+ callbacks = kwargs.get("callbacks", [])
103
+
104
+ # Build system prompt with tools
105
+ params = {"INSTRUCTION": self._config.instruction}
106
+ params.update(self._config.template_vars)
107
+
108
+ # Build system prompt using available tools
109
+ system_prompt = build_system_prompt(
110
+ system_prompt_template=self._config.system_prompt,
111
+ tool_prompt_template=self._config.tools_prompt,
112
+ tools=self._tools,
113
+ **params
114
+ )
115
+
116
+ # Process input message
117
+ if isinstance(message, (list, tuple)):
118
+ message = "\n".join(message)
119
+
120
+ # Implement your custom agent logic here
121
+ response = await self._custom_reasoning_loop(
122
+ system_prompt, message, tracer, callbacks
123
+ )
124
+
125
+ return AgentResponse(
126
+ name=self._name,
127
+ class_name=self.__class__.__name__,
128
+ response=response,
129
+ trace_id=tracer.trace_id
130
+ )
131
+
132
+ async def _custom_reasoning_loop(
133
+ self,
134
+ system_prompt: str,
135
+ user_message: str,
136
+ tracer: Tracer,
137
+ callbacks: List
138
+ ) -> str:
139
+ """Implement your custom reasoning logic."""
140
+
141
+ # Example: Multi-step reasoning with tool calls
142
+ messages = [
143
+ {"role": "system", "content": system_prompt},
144
+ {"role": "user", "content": user_message}
145
+ ]
146
+
147
+ for iteration in range(self._config.max_iterations):
148
+ # Generate LLM response
149
+ llm_response = await self._llm.generate_async(
150
+ messages=messages,
151
+ tracer=tracer,
152
+ callbacks=callbacks,
153
+ remote_mcp=self.get_remote_mcp_list()
154
+ )
155
+
156
+ # Check if tool calling is needed
157
+ if self._should_call_tool(llm_response):
158
+ try:
159
+ tool_result = await self.call_tool(
160
+ llm_response, tracer=tracer, callbacks=callbacks
161
+ )
162
+
163
+ # Add tool result to conversation
164
+ messages.append({"role": "assistant", "content": llm_response})
165
+ messages.append({"role": "user", "content": f"Tool result: {tool_result}"})
166
+
167
+ except Exception as e:
168
+ # Handle tool call errors
169
+ error_msg = f"Tool call failed: {str(e)}"
170
+ messages.append({"role": "user", "content": error_msg})
171
+ else:
172
+ # Return final response
173
+ return llm_response
174
+
175
+ return "Maximum iterations reached"
176
+
177
+ def _should_call_tool(self, response: str) -> bool:
178
+ """Determine if the response contains a tool call."""
179
+ # Implement your logic to detect tool calls
180
+ # This is a simple example - you might want more sophisticated parsing
181
+ try:
182
+ import json
183
+ parsed = json.loads(response.strip())
184
+ return "server" in parsed and "tool" in parsed and "arguments" in parsed
185
+ except:
186
+ return False
187
+
188
+ async def _cleanup(self):
189
+ """Optional: Cleanup custom resources."""
190
+ # Clean up any resources your agent created
191
+ self._custom_memory.clear()
192
+ ```
193
+
194
+ ### Step 3: Create Custom Prompt Templates
195
+
196
+ Create Jinja2 templates for your agent's prompts:
197
+
198
+ **custom_system_prompt.j2:**
199
+ ```jinja2
200
+ You are a specialized AI agent designed for {{INSTRUCTION}}.
201
+
202
+ {% if TOOLS_PROMPT is defined and TOOLS_PROMPT|length %}
203
+ {{TOOLS_PROMPT}}
204
+
205
+ When you need to use tools, respond with this JSON format:
206
+ {
207
+ "server": "server-name",
208
+ "tool": "tool-name",
209
+ "arguments": {"key": "value"}
210
+ }
211
+ {% endif %}
212
+
213
+ Follow these guidelines:
214
+ 1. Be thorough in your analysis
215
+ 2. Use tools when additional information is needed
216
+ 3. Provide clear, actionable responses
217
+ 4. If uncertain, ask clarifying questions
218
+ ```
219
+
220
+ ### Step 4: Register Your Agent
221
+
222
+ Create an `__init__.py` file or add to existing agent module:
223
+
224
+ ```python
225
+ from .my_custom_agent import MyCustomAgent
226
+
227
+ # The agent will be automatically registered due to the metaclass
228
+ __all__ = [..., "MyCustomAgent"]
229
+ ```
230
+
231
+ ## Testing Your Custom Agent
232
+
233
+ Create tests for your agent:
234
+
235
+ ```python
236
+ import pytest
237
+ from mcpuniverse.agent.my_custom_agent import MyCustomAgent
238
+ from mcpuniverse.llm.manager import ModelManager
239
+ from mcpuniverse.mcp.manager import MCPManager
240
+
241
+ @pytest.mark.asyncio
242
+ async def test_custom_agent():
243
+ # Setup
244
+ agent = MyCustomAgent(
245
+ mcp_manager=MCPManager(),
246
+ llm=ModelManager().build_model(name="openai"),
247
+ config={"name": "test-agent", "instruction": "Test agent"}
248
+ )
249
+
250
+ # Test initialization
251
+ await agent.initialize()
252
+ # Test execution
253
+ response = await agent.execute(message="Hello, world!")
254
+ assert response.name == "test-agent"
255
+ assert isinstance(response.response, str)
256
+ # Cleanup
257
+ await agent.cleanup()
258
+ ```
259
+
260
+ ## Best Practices
261
+
262
+ 1. **Configuration Management**: Use YAML files for configuration and support environment variable substitution
263
+ 2. **Error Handling**: Implement comprehensive error handling with meaningful error messages
264
+ 3. **Logging**: Use the framework's logging system for debugging and monitoring
265
+ 4. **Resource Cleanup**: Always implement proper cleanup in the `_cleanup` method
266
+ 6. **Memory Management**: Consider memory usage for long-running agents
267
+ 7. **Testing**: Write comprehensive tests for your agent's functionality
268
+ 8. **Documentation**: Document your agent's capabilities, configuration options, and usage examples
269
+
270
+ ## Troubleshooting
271
+
272
+ ### Common Issues
273
+
274
+ 1. **Agent Not Registered**: Ensure your agent class has the correct metaclass and is imported
275
+ 2. **Tool Not Found**: Check that MCP servers are properly configured and tools are available
276
+ 3. **Configuration Errors**: Validate YAML configuration files and required environment variables
277
+
278
+ ### Debugging
279
+
280
+ Use the built-in tracing system:
281
+
282
+ ```python
283
+ from mcpuniverse.tracer import Tracer
284
+
285
+ tracer = Tracer()
286
+ response = await agent.execute("test message", tracer=tracer)
287
+
288
+ # Examine trace data
289
+ trace_data = tracer.get_trace()
290
+ print(json.dumps(trace_data, indent=2))
291
+ ```
292
+
293
+ ## Conclusion
294
+
295
+ The MCPUniverse framework provides a powerful foundation for building custom AI agents. By following this guide, you can create sophisticated agents that leverage MCP tools, implement custom reasoning patterns, and integrate seamlessly with the broader MCPUniverse ecosystem.
296
+
297
+ For more examples and advanced patterns, refer to the existing agent implementations in the `mcpuniverse/agent/` directory.
docs/custom-evaluators-guide.md ADDED
@@ -0,0 +1,260 @@
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
1
+ # Custom Evaluators Implementation Guide
2
+
3
+ This guide provides comprehensive documentation for implementing custom evaluators in MCPUniverse. Evaluators are essential components that assess agent performance against specific criteria and validation rules.
4
+
5
+ ## Table of Contents
6
+
7
+ 1. [Evaluator System Overview](#evaluator-system-overview)
8
+ 2. [Architecture and Interface](#architecture-and-interface)
9
+ 3. [Function Types and Decorators](#function-types-and-decorators)
10
+ 4. [Implementation Steps](#implementation-steps)
11
+ 5. [Evaluation Functions](#evaluation-functions)
12
+ 6. [Comparison Functions](#comparison-functions)
13
+
14
+ ## Evaluator System Overview
15
+
16
+ The evaluator system is designed to validate agent outputs against predefined criteria. It consists of two main function types:
17
+
18
+ 1. **Evaluation Functions**: Transform and extract data from agent responses
19
+ 2. **Comparison Functions**: Compare processed data against expected values
20
+
21
+ ### Key Components
22
+
23
+ - **Evaluator Class**: Main evaluation orchestrator
24
+ - **EvaluatorConfig**: Configuration specification for evaluation rules
25
+ - **EvaluationResult**: Output containing evaluation results and reasoning
26
+ - **FunctionResult**: Wrapper for function outputs in the evaluation pipeline
27
+
28
+ ### System Architecture
29
+
30
+ ```
31
+ Agent Output → Evaluation Functions → Comparison Functions → EvaluationResult
32
+ ↓ ↓ ↓ ↓
33
+ Raw JSON Data Extraction Value Validation Pass/Fail + Reason
34
+ ```
35
+
36
+ ## Architecture and Interface
37
+
38
+ ### Core Classes
39
+
40
+ #### EvaluatorConfig
41
+ ```python
42
+ class EvaluatorConfig(BaseModel):
43
+ func: str # Function chain, e.g., "json -> get(key) -> len"
44
+ op: str = "" # Comparison operator, e.g., "=", "<", "contains"
45
+ value: Any = None # Expected value for comparison
46
+ op_args: Any = None # Additional arguments for comparison
47
+ desc: str = "" # Description for reporting
48
+ ```
49
+
50
+ #### EvaluationResult
51
+ ```python
52
+ class EvaluationResult(BaseModel):
53
+ config: EvaluatorConfig # Original configuration
54
+ response: str | Dict # Agent response being evaluated
55
+ passed: bool # Whether evaluation passed
56
+ reason: str = "" # Failure reason if applicable
57
+ error: str = "" # Error message if execution failed
58
+ ```
59
+
60
+ #### FunctionResult
61
+ ```python
62
+ class FunctionResult(BaseModel):
63
+ result: Any # The actual result data
64
+ ```
65
+
66
+ ### Evaluation Flow
67
+
68
+ 1. **Configuration Parsing**: Parse function chain from config
69
+ 2. **Function Execution**: Execute evaluation functions sequentially
70
+ 3. **Comparison**: Apply comparison operator with expected value
71
+ 4. **Result Generation**: Create EvaluationResult with pass/fail status
72
+
73
+ ## Function Types and Decorators
74
+
75
+ ### Evaluation Function Decorator
76
+
77
+ Use `@eval_func(name="function_name")` to register evaluation functions:
78
+
79
+ ```python
80
+ from mcpuniverse.evaluator.functions import eval_func, FunctionResult
81
+
82
+ @eval_func(name="my_custom_func")
83
+ async def my_custom_function(x: FunctionResult, *args, **kwargs) -> FunctionResult:
84
+ """Custom evaluation function."""
85
+ # Process the input and return FunctionResult
86
+ processed_data = process_data(x.result)
87
+ return FunctionResult(result=processed_data)
88
+ ```
89
+
90
+ ### Comparison Function Decorator
91
+
92
+ Use `@compare_func(name="comparison_name")` to register comparison functions:
93
+
94
+ ```python
95
+ from mcpuniverse.evaluator.functions import compare_func
96
+
97
+ @compare_func(name="my_custom_comparison")
98
+ async def my_custom_comparison(a: Any, b: Any, *args, **kwargs) -> tuple[bool, str]:
99
+ """Custom comparison function."""
100
+ # Compare values and return (success, reason)
101
+ if custom_condition(a, b):
102
+ return True, ""
103
+ return False, "Custom validation failed"
104
+ ```
105
+
106
+ ## Implementation Steps
107
+
108
+ ### Step 1: Create Module Structure
109
+
110
+ Create a new evaluator module in the appropriate domain:
111
+
112
+ ```bash
113
+ mkdir mcpuniverse/evaluator/my_domain
114
+ touch mcpuniverse/evaluator/my_domain/__init__.py
115
+ touch mcpuniverse/evaluator/my_domain/functions.py
116
+ ```
117
+
118
+ ### Step 2: Implement Evaluation Functions
119
+
120
+ ```python
121
+ # mcpuniverse/evaluator/my_domain/functions.py
122
+ """
123
+ Evaluation functions for my custom domain
124
+ """
125
+ import json
126
+ from typing import Any
127
+ from mcpuniverse.evaluator.functions import eval_func, compare_func, FunctionResult
128
+
129
+ @eval_func(name="extract_score")
130
+ async def extract_score(x: FunctionResult, *args, **kwargs) -> FunctionResult:
131
+ """Extract numerical score from response."""
132
+ if isinstance(x, FunctionResult):
133
+ data = x.result
134
+ if isinstance(data, dict) and 'score' in data:
135
+ return FunctionResult(result=float(data['score']))
136
+ elif isinstance(data, str):
137
+ # Try to extract number from string
138
+ import re
139
+ match = re.search(r'\d+\.?\d*', data)
140
+ if match:
141
+ return FunctionResult(result=float(match.group()))
142
+ raise ValueError("Could not extract score from input")
143
+
144
+ @eval_func(name="normalize_text")
145
+ async def normalize_text(x: FunctionResult, *args, **kwargs) -> FunctionResult:
146
+ """Normalize text for comparison."""
147
+ if isinstance(x, FunctionResult):
148
+ text = str(x.result).lower().strip()
149
+ # Remove extra whitespace
150
+ normalized = ' '.join(text.split())
151
+ return FunctionResult(result=normalized)
152
+ raise ValueError("Could not normalize text")
153
+ ```
154
+
155
+ ### Step 3: Implement Comparison Functions
156
+
157
+ ```python
158
+ @compare_func(name="score_threshold")
159
+ async def score_threshold(a: Any, b: Any, *args, **kwargs) -> tuple[bool, str]:
160
+ """Check if score meets threshold."""
161
+ if isinstance(a, FunctionResult):
162
+ a = a.result
163
+ if isinstance(b, FunctionResult):
164
+ b = b.result
165
+
166
+ threshold = float(b)
167
+ score = float(a)
168
+
169
+ if score >= threshold:
170
+ return True, ""
171
+ return False, f"Score {score} below threshold {threshold}"
172
+
173
+ @compare_func(name="text_similarity")
174
+ async def text_similarity(a: Any, b: Any, *args, **kwargs) -> tuple[bool, str]:
175
+ """Check text similarity using fuzzy matching."""
176
+ from difflib import SequenceMatcher
177
+
178
+ if isinstance(a, FunctionResult):
179
+ a = a.result
180
+ if isinstance(b, FunctionResult):
181
+ b = b.result
182
+
183
+ similarity = SequenceMatcher(None, str(a), str(b)).ratio()
184
+ threshold = 0.8 # Default threshold
185
+
186
+ if len(args) > 2 and args[2]: # op_args provided
187
+ threshold = float(args[2].get('threshold', 0.8))
188
+
189
+ if similarity >= threshold:
190
+ return True, ""
191
+ return False, f"Text similarity {similarity:.2f} below threshold {threshold}"
192
+ ```
193
+
194
+ ### Step 4: Register Functions in Module
195
+
196
+ Update the main evaluator `__init__.py`:
197
+
198
+ ```python
199
+ # mcpuniverse/evaluator/__init__.py
200
+ from .functions import *
201
+ from .my_domain.functions import * # Add your module
202
+
203
+ __all__ = [
204
+ "Evaluator",
205
+ "EvaluationResult",
206
+ "EvaluatorConfig"
207
+ ]
208
+ ```
209
+
210
+ ## Built-in Evaluation Functions
211
+
212
+ | Function | Purpose | Usage Example |
213
+ |----------|---------|---------------|
214
+ | `json` | Parse JSON string | `"json"` |
215
+ | `get(key)` | Extract dictionary value | `"json -> get(city)"` |
216
+ | `len` | Get array/string length | `"json -> get(items) -> len"` |
217
+ | `foreach` | Iterate over arrays | `"json -> get(routes) -> foreach -> get(name)"` |
218
+ | `raw` | Pass through data unchanged | `"raw"` |
219
+
220
+ ## Built-in Comparison Functions
221
+
222
+ | Function | Purpose | Usage Example |
223
+ |----------|---------|---------------|
224
+ | `=` | Exact equality | `"op": "=", "value": "expected"` |
225
+ | `<`, `>`, `<=`, `>=` | Numerical comparison | `"op": ">", "value": 100` |
226
+ | `in` | Membership test | `"op": "in", "value": ["a", "b", "c"]` |
227
+ | `contain` | Contains test | `"op": "contain", "value": "substring"` |
228
+
229
+ ## Task Configuration Example
230
+
231
+ ```json
232
+ {
233
+ "category": "ecommerce",
234
+ "question": "Calculate the final price for a shopping cart with discount",
235
+ "mcp_servers": [{"name": "ecommerce-api"}],
236
+ "output_format": {
237
+ "original_price": "<original total>",
238
+ "discount_percentage": "<discount %age>",
239
+ "discounted_price": "<final price>",
240
+ "savings": "<amount saved>"
241
+ },
242
+ "evaluators": [
243
+ {
244
+ "func": "json",
245
+ "op": "validate_discount",
246
+ "op_args": {
247
+ "discount_percentage": 15,
248
+ "tolerance": 0.5
249
+ }
250
+ },
251
+ {
252
+ "func": "json -> extract_order_total",
253
+ "op": ">",
254
+ "value": 0
255
+ }
256
+ ]
257
+ }
258
+ ```
259
+
260
+ This guide provides a complete framework for implementing custom evaluators in MCPUniverse. Follow these patterns and best practices to create robust, maintainable evaluation functions that accurately assess agent performance in your specific domain.
docs/python-sandbox-setup.md ADDED
@@ -0,0 +1,33 @@
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
1
+ # Python Sandbox MCP Server Setup Guide
2
+
3
+ This guide will help you set up the python sandbox MCP server integration within the MCP-Universe project.
4
+
5
+ ### 1. Build the Docker Image
6
+
7
+ First, build the Docker image that will run the HTTP server inside the container:
8
+
9
+ ```bash
10
+ docker build -f docker/python_code_sandbox/Dockerfile.server -t python-code-sandbox:latest .
11
+ ```
12
+
13
+
14
+
15
+ ### 2. Start the Container Manually
16
+
17
+ Start the Docker container manually with the following command:
18
+
19
+ ```bash
20
+ docker run -d \
21
+ --name python-sandbox-server \
22
+ -p 18080:8080 \
23
+ -e SANDBOX_PORT=8080 \
24
+ -e SANDBOX_TEMP_DIR=/tmp/sandbox_executions \
25
+ --memory=100g \
26
+ --cpus=20 \
27
+ python-code-sandbox:latest
28
+ ```
29
+
30
+ **Verify the container is running:**
31
+ ```bash
32
+ docker logs -f --tail 10 python-sandbox-server
33
+ ```
docs/system-architecture.md ADDED
@@ -0,0 +1,275 @@
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
1
+ # MCPUniverse System Architecture
2
+
3
+ This document provides a comprehensive overview of the MCPUniverse system architecture, including its core components, design patterns, and how the different layers interact to provide a framework for developing and benchmarking AI agents using the Model Control Protocol (MCP).
4
+
5
+ ## Overview
6
+
7
+ MCPUniverse is a modular framework designed to facilitate AI agent development, testing, and benchmarking. The system is built around the Model Control Protocol (MCP) standard, which enables agents to interact with external tools and services in a standardized way.
8
+
9
+ ## High-Level Architecture
10
+
11
+ ```
12
+ ┌─────────────────────────────────────────────────────────────────┐
13
+ │ Application Layer │
14
+ ├─────────────────────────────────────────────────────────────────┤
15
+ │ Dashboard │ Web API │ Python Lib │ Benchmarks │
16
+ │ (Gradio) │ (FastAPI) │ │ │
17
+ └─────────────┬─────────────────┬─────────────────┬───────────────┘
18
+ │ │ │
19
+ ┌─────────────▼─────────────────▼─────────────────▼──────────────┐
20
+ │ Orchestration Layer │
21
+ ├────────────────────────────────────────────────────────────────┤
22
+ │ Workflows │ Benchmark Runner │
23
+ │ (Chain, Router, etc.) │ (Evaluation Engine) │
24
+ └─────────────┬─────────────────┬─────────────────┬──────────────┘
25
+ │ │ │
26
+ ┌─────────────▼─────────────────▼─────────────────▼──────────────┐
27
+ │ Agent Layer │
28
+ ├────────────────────────────────────────────────────────────────┤
29
+ │ BasicAgent │ ReActAgent │ FunctionCall │ Other │
30
+ │ │ │ Agent │ Agents │
31
+ └─────────────┬─────────────────┬─────────────────┬──────────────┘
32
+ │ │ │
33
+ ┌─────────────▼─────────────────▼─────────────────▼──────────────┐
34
+ │ Foundation Layer │
35
+ ├────────────────────────────────────────────────────────────────┤
36
+ │ MCP Manager │ LLM Manager │ Memory Systems │ Tracers │
37
+ │ (Servers & │ (OpenAI, │ (RAM, Redis) │ │
38
+ │ Clients) │ Claude, etc.) │ │ │
39
+ └─────────────────┴─────────────────┴─────────────────┴──────────┘
40
+ ```
41
+
42
+ ## Core Components
43
+
44
+ ### 1. Agent Layer (`mcpuniverse/agent/`)
45
+
46
+ The agent layer is the core of MCPUniverse, providing different types of AI agents that can reason, act, and interact with external tools.
47
+
48
+ #### BaseAgent
49
+ - **Purpose**: Abstract base class that all agents inherit from
50
+ - **Key Features**:
51
+ - MCP server connection management
52
+ - Tool execution capabilities
53
+ - Configuration management
54
+ - Tracing and debugging support
55
+ - Lifecycle management (initialize, execute, cleanup)
56
+
57
+ #### Agent Types
58
+ - **BasicAgent**: Simple LLM interaction agent for straightforward tasks
59
+ - **ReActAgent**: Implements reasoning and acting pattern with iterative thinking
60
+ - **FunctionCallAgent**: Uses native LLM tool calling APIs
61
+ - **ReflectionAgent**: Self-reflective agent with memory capabilities
62
+ - **ClaudeCodeAgent**: Specialized agent for code-related tasks
63
+
64
+ #### Key Interfaces
65
+ ```python
66
+ class Executor:
67
+ async def execute(message, **kwargs) -> AgentResponse
68
+ async def initialize()
69
+ async def cleanup()
70
+ def set_name(name: str)
71
+ ```
72
+
73
+ ### 2. MCP Layer (`mcpuniverse/mcp/`)
74
+
75
+ Manages Model Control Protocol servers and clients, enabling agents to interact with external tools and services.
76
+
77
+ #### MCPManager
78
+ - **Purpose**: Central management of MCP server configurations and client connections
79
+ - **Key Features**:
80
+ - Server configuration loading from JSON
81
+ - Dynamic server registration and management
82
+ - Client building for stdio and SSE transports
83
+ - Environment variable templating
84
+ - Parameter validation
85
+
86
+ #### MCPClient
87
+ - **Purpose**: Handles connections to individual MCP servers
88
+ - **Supported Transports**: stdio, SSE (Server-Sent Events)
89
+ - **Operations**: Tool execution, resource access, server communication
90
+
91
+ #### Example built-in Servers
92
+ - **Weather**: National Weather Service API integration
93
+ - **Google Search**: Search functionality via SERP API
94
+ - **Google Sheets**: Spreadsheet operations
95
+ - **Wikipedia**: Knowledge base access
96
+ - **Blender**: 3D modeling operations
97
+ - **Yahoo Finance**: Financial data access
98
+
99
+ ### 3. LLM Layer (`mcpuniverse/llm/`)
100
+
101
+ Provides unified interface to multiple language model providers.
102
+
103
+ #### BaseLLM
104
+ - **Purpose**: Abstract base class for all LLM implementations
105
+ - **Key Features**:
106
+ - Async generation capabilities
107
+ - Message handling and formatting
108
+ - Context management
109
+ - Configuration export/import
110
+
111
+ #### Supported Providers
112
+ - **OpenAI**: OpenAI models
113
+ - **Anthropic**: Claude models
114
+ - **Google**: Gemini models
115
+ - **Mistral**: Mistral AI models
116
+ - **Ollama**: Local model serving
117
+ - **Grok**: xAI's Grok models
118
+ - **DeepSeek**: DeepSeek models
119
+
120
+ ### 4. Workflow Layer (`mcpuniverse/workflows/`)
121
+
122
+ Orchestrates complex multi-agent interactions and task execution patterns.
123
+
124
+ #### BaseWorkflow
125
+ - **Purpose**: Foundation for workflow implementations
126
+ - **Key Features**:
127
+ - Agent lifecycle management
128
+ - Execution coordination
129
+ - Result aggregation
130
+ - Error handling and recovery
131
+
132
+ #### Workflow Types
133
+ - **Chain**: Sequential agent execution
134
+ - **Router**: Conditional agent selection based on input
135
+ - **Parallelization**: Concurrent agent execution
136
+ - **Orchestrator**: Complex multi-agent coordination
137
+ - **EvaluatorOptimizer**: Agent performance optimization
138
+
139
+ ### 5. Benchmark Layer (`mcpuniverse/benchmark/`)
140
+
141
+ Comprehensive system for evaluating agent performance across different tasks and domains.
142
+
143
+ #### BenchmarkRunner
144
+ - **Purpose**: Executes benchmarks and collects performance data
145
+ - **Key Features**:
146
+ - YAML-based configuration
147
+ - Task definition and execution
148
+ - Result collection and analysis
149
+ - Performance metrics calculation
150
+
151
+ #### Task System
152
+ - **Task Definition**: JSON-based task specifications
153
+ - **Evaluation**: Custom evaluators for different domains
154
+ - **Metrics**: Success rates, execution time, resource usage
155
+
156
+ #### Supported Domains
157
+ - **Google Maps**: Location and navigation tasks
158
+ - **GitHub**: Repository and code management
159
+ - **Blender**: 3D modeling and rendering
160
+ - **Playwright**: Web automation
161
+ - **Financial**: Yahoo Finance integration
162
+ - **Multi-server**: Complex cross-domain tasks
163
+
164
+ ### 6. Evaluation Layer (`mcpuniverse/evaluator/`)
165
+
166
+ Domain-specific evaluation functions for assessing agent performance.
167
+
168
+ #### Evaluator System
169
+ - **Purpose**: Automated assessment of agent outputs
170
+ - **Key Features**:
171
+ - JSON-based output validation
172
+ - Custom evaluation functions
173
+ - Domain-specific scoring
174
+ - Comparative analysis
175
+
176
+ ### 7. Tracing Layer (`mcpuniverse/tracer/`)
177
+
178
+ Comprehensive debugging and monitoring system for agent execution.
179
+
180
+ #### Tracer
181
+ - **Purpose**: Execution tracking and debugging
182
+ - **Key Features**:
183
+ - Hierarchical trace collection
184
+ - Performance monitoring
185
+ - Error tracking
186
+ - Execution replay
187
+
188
+ #### Collectors
189
+ - **Memory**: In-memory trace storage
190
+ - **File**: Persistent file-based storage
191
+ - **SQLite**: Database-backed trace storage
192
+
193
+ ### 8. Memory Layer (`mcpuniverse/agent/memory/`)
194
+
195
+ Provides memory capabilities for agents to maintain context across interactions.
196
+
197
+ #### Memory Types
198
+ - **Short-term Memory**:
199
+ - **RAM**: In-memory storage for session data
200
+ - **Redis**: Distributed memory for scalable deployments
201
+ - **Context Management**: Conversation and task context preservation
202
+
203
+ ### 9. Callback System (`mcpuniverse/callbacks/`)
204
+
205
+ Event-driven system for monitoring and reacting to agent execution events.
206
+
207
+ #### Callback Types
208
+ - **Status Updates**: Execution state changes
209
+ - **Error Handling**: Exception and failure notifications
210
+ - **Performance Monitoring**: Metrics and timing data
211
+ - **Custom Handlers**: Application-specific event processing
212
+
213
+ ### 10. Application Layer (`mcpuniverse/app/`)
214
+
215
+ Web application interface providing REST APIs and user interface.
216
+
217
+ #### Web API (FastAPI)
218
+ - **Endpoints**: Agent management, benchmark execution, task monitoring
219
+ - **Authentication**: Token-based security
220
+ - **Database**: PostgreSQL for persistent storage
221
+ - **Background Tasks**: Celery for async processing
222
+
223
+ #### Dashboard (Gradio)
224
+ - **Purpose**: Interactive web interface for agent testing
225
+ - **Features**: Real-time execution monitoring, result visualization
226
+
227
+ ## Data Flow
228
+
229
+ ### 1. Agent Execution Flow
230
+ ```
231
+ User Input → Agent.execute() → LLM.generate() → Tool Execution → Response
232
+ ↓ ↓ ↓ ↓ ↓
233
+ Callbacks ← Tracer ← Callbacks ← MCP Client ← Evaluation
234
+ ```
235
+
236
+ ### 2. MCP Tool Execution Flow
237
+ ```
238
+ Agent → MCPManager → MCPClient → MCP Server → External Service
239
+ ↑ ↑ ↑ ↑ ↑
240
+ Config Server Transport Tool Call API/Service
241
+ Loading Selection Protocol Execution Response
242
+ ```
243
+
244
+ ### 3. Benchmark Execution Flow
245
+ ```
246
+ YAML Config → BenchmarkRunner → Task Execution → Evaluation → Results
247
+ ↓ ↓ ↓ ↓ ↓
248
+ Task Def Agent Creation Agent Execution Scoring Report Gen
249
+ ```
250
+
251
+ ## Configuration Management
252
+
253
+ ### 1. Hierarchical Configuration
254
+ - **Global**: Framework-level settings
255
+ - **Agent**: Agent-specific configurations
256
+ - **Server**: MCP server configurations
257
+ - **Task**: Benchmark task definitions
258
+
259
+ ### 2. Environment Variable Templating
260
+ ```json
261
+ {
262
+ "env": {
263
+ "API_KEY": "{{MY_API_KEY}}",
264
+ "PORT": "{{PORT}}"
265
+ }
266
+ }
267
+ ```
268
+
269
+ ### 3. Dynamic Configuration
270
+ Runtime configuration updates through the MCPManager API:
271
+ ```python
272
+ manager.add_server_config("new-server", config)
273
+ ```
274
+
275
+ This architecture provides a solid foundation for AI agent development while maintaining flexibility, scalability, and extensibility. The modular design allows developers to focus on specific aspects of agent behavior while leveraging the robust infrastructure for common operations like tool execution, memory management, and performance evaluation.
docs/technical-blog.md ADDED
@@ -0,0 +1,298 @@
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
1
+ # MCP-Universe: A Comprehensive Framework for AI Agent Development and Benchmarking
2
+
3
+ The landscape of AI agent development has evolved rapidly, with developers needing robust frameworks to build, test, and benchmark intelligent systems. **MCP-Universe** emerges as a comprehensive solution, providing a modular framework designed around the Model Control Protocol (MCP) standard for developing, orchestrating, and evaluating AI agents at scale.
4
+
5
+ ## The Vision Behind MCP-Universe
6
+
7
+ Traditional AI agent development often suffers from fragmented tooling, inconsistent interfaces, and limited benchmarking capabilities. MCP-Universe addresses these challenges by providing:
8
+
9
+ - **Unified Tool Integration**: Standardized connections to external services through MCP
10
+ - **Multi-Model Support**: Provider-agnostic LLM integration across OpenAI, Anthropic, Google, and more
11
+ - **Flexible Agent Architectures**: From simple function-calling to complex reasoning patterns
12
+ - **Comprehensive Benchmarking**: Automated evaluation across diverse domains and tasks
13
+ - **Scalable Orchestration**: Multi-agent workflows and coordination patterns
14
+
15
+ ## Core Architecture: Built for Scale and Flexibility
16
+
17
+ ### Layered Architecture Design
18
+
19
+ MCP-Universe follows a carefully designed layered architecture that separates concerns while maintaining flexibility:
20
+
21
+ ```
22
+ ┌─────────────────────────────────────────────────────────────────┐
23
+ │ Application Layer │
24
+ ├─────────────────────────────────────────────────────────────────┤
25
+ │ Dashboard │ Web API │ Python Lib │ Benchmarks │
26
+ │ (Gradio) │ (FastAPI) │ │ │
27
+ └─────────────┬─────────────────┬─────────────────┬───────────────┘
28
+ │ │ │
29
+ ┌─────────────▼─────────────────▼─────────────────▼──────────────┐
30
+ │ Orchestration Layer │
31
+ ├────────────────────────────────────────────────────────────────┤
32
+ │ Workflows │ Benchmark Runner │
33
+ │ (Chain, Router, etc.) │ (Evaluation Engine) │
34
+ └─────────────┬─────────────────┬─────────────────┬──────────────┘
35
+ │ │ │
36
+ ┌─────────────▼─────────────────▼─────────────────▼──────────────┐
37
+ │ Agent Layer │
38
+ ├────────────────────────────────────────────────────────────────┤
39
+ │ BasicAgent │ ReActAgent │ FunctionCall │ Other │
40
+ │ │ │ Agent │ Agents │
41
+ └─────────────┬─────────────────┬────────────────┬───────────────┘
42
+ │ │ │
43
+ ┌─────────────▼─────────────────▼────────────────▼───────────────┐
44
+ │ Foundation Layer │
45
+ ├────────────────────────────────────────────────────────────────┤
46
+ │ MCP Manager │ LLM Manager │ Memory Systems │ Tracers │
47
+ │ (Servers & │ (OpenAI, │ (RAM, Redis) │ │
48
+ │ Clients) │ Claude, etc.) │ │ │
49
+ └─────────────────┴─────────────────┴─────────────────┴──────────┘
50
+ ```
51
+
52
+ This architecture provides several key benefits:
53
+
54
+ - **Modularity**: Each layer can be developed and tested independently
55
+ - **Extensibility**: New components can be added without affecting existing functionality
56
+ - **Scalability**: The design supports everything from single-agent tasks to complex multi-agent orchestration
57
+ - **Maintainability**: Clear separation of concerns makes the system easier to debug and extend
58
+
59
+ ### The MCP Foundation
60
+
61
+ At its core, MCP-Universe leverages the **Model Control Protocol (MCP)**, which standardizes how AI agents interact with external tools and services. This provides:
62
+
63
+ - **Unified Interface**: Consistent API across different tool types
64
+ - **Transport Flexibility**: Support for both stdio and Server-Sent Events (SSE) communication
65
+ - **Dynamic Tool Discovery**: Runtime discovery and registration of capabilities
66
+ - **Standardized Error Handling**: Consistent error reporting across all tools
67
+
68
+ ## Key Designs
69
+
70
+ ### 1. Agent Architecture Variety
71
+
72
+ MCP-Universe supports multiple agent reasoning patterns, each optimized for different use cases, e.g.:
73
+
74
+ #### **FunctionCallAgent** - Efficient Tool Usage
75
+ Leverages native LLM tool calling APIs for optimal performance:
76
+ ```yaml
77
+ kind: agent
78
+ spec:
79
+ name: function-agent
80
+ type: function-call
81
+ config:
82
+ llm: gpt-4o-llm
83
+ instruction: You can call functions to help users.
84
+ servers:
85
+ - name: weather
86
+ - name: google-maps
87
+ ```
88
+
89
+ #### **ReActAgent** - Reasoning and Acting
90
+ Implements the ReAct pattern for complex problem-solving:
91
+ ```yaml
92
+ kind: agent
93
+ spec:
94
+ name: reasoning-agent
95
+ type: react
96
+ config:
97
+ llm: gpt-4o-llm
98
+ instruction: You are a ReAct agent that reasons and acts.
99
+ max_iterations: 10
100
+ servers:
101
+ - name: weather
102
+ - name: google-search
103
+ ```
104
+
105
+ #### **ReflectionAgent** - Self-Improving
106
+ Uses reflection for enhanced reasoning and learning:
107
+ ```yaml
108
+ kind: agent
109
+ spec:
110
+ name: reflective-agent
111
+ type: reflection
112
+ config:
113
+ llm: gpt-4o-llm
114
+ instruction: You improve through self-reflection.
115
+ max_iterations: 5
116
+ ```
117
+
118
+ ### 2. Workflow Orchestration
119
+
120
+ Beyond individual agents, MCP-Universe provides sophisticated workflow patterns, e.g.:
121
+
122
+ #### **Chain Workflows** - Sequential Processing
123
+ Execute agents in sequence, passing results between them:
124
+ ```yaml
125
+ kind: workflow
126
+ spec:
127
+ name: analysis-chain
128
+ type: chain
129
+ config:
130
+ agents:
131
+ - data-collector
132
+ - data-analyzer
133
+ - report-generator
134
+ ```
135
+
136
+ #### **Orchestrator Workflows** - Complex Coordination
137
+ Plan and coordinate multiple agents for complex tasks:
138
+ ```yaml
139
+ kind: workflow
140
+ spec:
141
+ name: research-orchestrator
142
+ type: orchestrator
143
+ config:
144
+ llm: gpt-4o-llm
145
+ agents:
146
+ - researcher
147
+ - analyst
148
+ - writer
149
+ plan_type: "full"
150
+ max_iterations: 10
151
+ ```
152
+
153
+ ### 3. Comprehensive Benchmarking System
154
+
155
+ MCP-Universe's benchmarking capabilities set it apart from other frameworks:
156
+
157
+ #### **Multi-Domain Evaluation**
158
+ Support for diverse domains, including but not limited to:
159
+ - **Google Maps**: Location and navigation tasks
160
+ - **GitHub**: Repository management and code analysis
161
+ - **Blender**: 3D modeling and rendering operations
162
+ - **Web Automation**: Playwright-based browser interactions
163
+ - **Financial Services**: Yahoo Finance integration
164
+ - **Multi-server Tasks**: Complex cross-domain scenarios
165
+
166
+ #### **Flexible Evaluation Functions**
167
+ JSON-based evaluation with chainable functions:
168
+ ```json
169
+ {
170
+ "evaluators": [
171
+ {
172
+ "func": "json -> get(forecast) -> len",
173
+ "op": ">",
174
+ "value": 3
175
+ },
176
+ {
177
+ "func": "json -> get(forecast) -> foreach -> get(day)",
178
+ "op": "contains",
179
+ "value": "Monday"
180
+ }
181
+ ]
182
+ }
183
+ ```
184
+
185
+ #### **Custom Evaluator Support**
186
+ Create domain-specific evaluation functions:
187
+ ```python
188
+ @eval_func(name="extract_score")
189
+ async def extract_score(x: FunctionResult, *args, **kwargs) -> FunctionResult:
190
+ """Extract numerical score from response."""
191
+ # Custom evaluation logic
192
+ return FunctionResult(result=processed_score)
193
+ ```
194
+
195
+ ## Key Benefits for Developers
196
+
197
+ ### 1. **Rapid Development**
198
+ - Pre-built agent types for common patterns
199
+ - YAML-based configuration for easy customization
200
+ - Rich ecosystem of MCP servers for immediate tool access
201
+ - Comprehensive documentation and examples
202
+
203
+ ### 2. **Production Ready**
204
+ - Built-in tracing and debugging capabilities
205
+ - Memory management with Redis support for scalability
206
+ - FastAPI-based web interface for monitoring and control
207
+ - Comprehensive error handling and recovery
208
+
209
+ ### 3. **Extensible Architecture**
210
+ - Plugin-based MCP server integration
211
+ - Custom agent type support
212
+ - Flexible evaluation system
213
+ - Multi-LLM provider support
214
+
215
+ ### 4. **Research Friendly**
216
+ - Comprehensive benchmarking suite
217
+ - Detailed execution tracing
218
+ - Performance metrics collection
219
+ - Comparative analysis tools
220
+
221
+ ## Getting Started: A Practical Example
222
+
223
+ To begin with MCP-Universe:
224
+
225
+ 1. **Clone the repository**
226
+ 2. **Set up your environment variables** in `.env` (copy from `.env.example`)
227
+ 3. **Install dependencies**: `pip install -r requirements.txt`
228
+
229
+ Here's how to create a weather analysis agent in MCP-Universe:
230
+
231
+ ### 1. Define Your LLM and Agent
232
+ ```yaml
233
+ kind: llm
234
+ spec:
235
+ name: gpt-4o-llm
236
+ type: openai
237
+ config:
238
+ model_name: gpt-4o
239
+ temperature: 0.1
240
+
241
+ ---
242
+ kind: agent
243
+ spec:
244
+ name: weather-analyst
245
+ type: react
246
+ config:
247
+ llm: gpt-4o-llm
248
+ instruction: You are a weather analysis expert.
249
+ max_iterations: 5
250
+ servers:
251
+ - name: weather
252
+ ```
253
+
254
+ ### 2. Create a Benchmark
255
+ ```yaml
256
+ kind: benchmark
257
+ spec:
258
+ description: Weather forecasting evaluation
259
+ agent: weather-analyst
260
+ tasks:
261
+ - weather/forecast_accuracy.json
262
+ - weather/multi_location_comparison.json
263
+ ```
264
+
265
+ ### 3. Run and Evaluate
266
+ ```python
267
+ import os
268
+ from mcpuniverse.tracer.collectors import MemoryCollector
269
+ from mcpuniverse.benchmark.runner import BenchmarkRunner
270
+
271
+ # Initialize components
272
+ trace_collector = MemoryCollector()
273
+ benchmark = BenchmarkRunner("weather_benchmark.yaml")
274
+
275
+ # Run benchmark
276
+ results = await benchmark.run(
277
+ trace_collector=trace_collector,
278
+ store_folder="<TMP-FOLDER>"
279
+ )
280
+ print(results)
281
+ ```
282
+
283
+ ## The Future of AI Agent Development
284
+
285
+ MCP-Universe represents a significant step forward in AI agent development frameworks. By providing:
286
+
287
+ - **Standardized Integration** through MCP
288
+ - **Flexible Architecture** supporting diverse agent types
289
+ - **Comprehensive Benchmarking** for rigorous evaluation
290
+ - **Production-Ready Infrastructure** for real-world deployment
291
+
292
+ It enables developers to focus on building intelligent behavior rather than managing infrastructure complexity.
293
+
294
+ Whether you're researching new agent architectures, building production AI systems, or benchmarking agent performance across domains, MCP-Universe provides the foundation you need to succeed in the rapidly evolving landscape of AI agent development.
295
+
296
+ ---
297
+
298
+ *MCP-Universe is actively maintained and welcomes contributions from the community. Visit our documentation and GitHub repository to get started building intelligent agents today.*
license_info.md ADDED
@@ -0,0 +1,251 @@
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
1
+ License Info
2
+ ------------
3
+
4
+ Most projects we open source should use the [Apache License v2](https://opensource.org/license/apache-2-0/) license. Samples, demos, and blog / doc code examples should instead use [CC-0](https://creativecommons.org/publicdomain/zero/1.0/). If you strongly feel your project should perhaps use a different license clause, please engage with legal team.
5
+
6
+ For the ALv2 license, create a `LICENSE.txt` file (or use the one in this template repo) in the root of your repo containing:
7
+ ```
8
+ Apache License Version 2.0
9
+
10
+ Copyright (c) 2024 Salesforce, Inc.
11
+ All rights reserved.
12
+
13
+ Apache License
14
+ Version 2.0, January 2004
15
+ http://www.apache.org/licenses/
16
+
17
+ TERMS AND CONDITIONS FOR USE, REPRODUCTION, AND DISTRIBUTION
18
+
19
+ 1. Definitions.
20
+
21
+ "License" shall mean the terms and conditions for use, reproduction,
22
+ and distribution as defined by Sections 1 through 9 of this document.
23
+
24
+ "Licensor" shall mean the copyright owner or entity authorized by
25
+ the copyright owner that is granting the License.
26
+
27
+ "Legal Entity" shall mean the union of the acting entity and all
28
+ other entities that control, are controlled by, or are under common
29
+ control with that entity. For the purposes of this definition,
30
+ "control" means (i) the power, direct or indirect, to cause the
31
+ direction or management of such entity, whether by contract or
32
+ otherwise, or (ii) ownership of fifty percent (50%) or more of the
33
+ outstanding shares, or (iii) beneficial ownership of such entity.
34
+
35
+ "You" (or "Your") shall mean an individual or Legal Entity
36
+ exercising permissions granted by this License.
37
+
38
+ "Source" form shall mean the preferred form for making modifications,
39
+ including but not limited to software source code, documentation
40
+ source, and configuration files.
41
+
42
+ "Object" form shall mean any form resulting from mechanical
43
+ transformation or translation of a Source form, including but
44
+ not limited to compiled object code, generated documentation,
45
+ and conversions to other media types.
46
+
47
+ "Work" shall mean the work of authorship, whether in Source or
48
+ Object form, made available under the License, as indicated by a
49
+ copyright notice that is included in or attached to the work
50
+ (an example is provided in the Appendix below).
51
+
52
+ "Derivative Works" shall mean any work, whether in Source or Object
53
+ form, that is based on (or derived from) the Work and for which the
54
+ editorial revisions, annotations, elaborations, or other modifications
55
+ represent, as a whole, an original work of authorship. For the purposes
56
+ of this License, Derivative Works shall not include works that remain
57
+ separable from, or merely link (or bind by name) to the interfaces of,
58
+ the Work and Derivative Works thereof.
59
+
60
+ "Contribution" shall mean any work of authorship, including
61
+ the original version of the Work and any modifications or additions
62
+ to that Work or Derivative Works thereof, that is intentionally
63
+ submitted to Licensor for inclusion in the Work by the copyright owner
64
+ or by an individual or Legal Entity authorized to submit on behalf of
65
+ the copyright owner. For the purposes of this definition, "submitted"
66
+ means any form of electronic, verbal, or written communication sent
67
+ to the Licensor or its representatives, including but not limited to
68
+ communication on electronic mailing lists, source code control systems,
69
+ and issue tracking systems that are managed by, or on behalf of, the
70
+ Licensor for the purpose of discussing and improving the Work, but
71
+ excluding communication that is conspicuously marked or otherwise
72
+ designated in writing by the copyright owner as "Not a Contribution."
73
+
74
+ "Contributor" shall mean Licensor and any individual or Legal Entity
75
+ on behalf of whom a Contribution has been received by Licensor and
76
+ subsequently incorporated within the Work.
77
+
78
+ 2. Grant of Copyright License. Subject to the terms and conditions of
79
+ this License, each Contributor hereby grants to You a perpetual,
80
+ worldwide, non-exclusive, no-charge, royalty-free, irrevocable
81
+ copyright license to reproduce, prepare Derivative Works of,
82
+ publicly display, publicly perform, sublicense, and distribute the
83
+ Work and such Derivative Works in Source or Object form.
84
+
85
+ 3. Grant of Patent License. Subject to the terms and conditions of
86
+ this License, each Contributor hereby grants to You a perpetual,
87
+ worldwide, non-exclusive, no-charge, royalty-free, irrevocable
88
+ (except as stated in this section) patent license to make, have made,
89
+ use, offer to sell, sell, import, and otherwise transfer the Work,
90
+ where such license applies only to those patent claims licensable
91
+ by such Contributor that are necessarily infringed by their
92
+ Contribution(s) alone or by combination of their Contribution(s)
93
+ with the Work to which such Contribution(s) was submitted. If You
94
+ institute patent litigation against any entity (including a
95
+ cross-claim or counterclaim in a lawsuit) alleging that the Work
96
+ or a Contribution incorporated within the Work constitutes direct
97
+ or contributory patent infringement, then any patent licenses
98
+ granted to You under this License for that Work shall terminate
99
+ as of the date such litigation is filed.
100
+
101
+ 4. Redistribution. You may reproduce and distribute copies of the
102
+ Work or Derivative Works thereof in any medium, with or without
103
+ modifications, and in Source or Object form, provided that You
104
+ meet the following conditions:
105
+
106
+ (a) You must give any other recipients of the Work or
107
+ Derivative Works a copy of this License; and
108
+
109
+ (b) You must cause any modified files to carry prominent notices
110
+ stating that You changed the files; and
111
+
112
+ (c) You must retain, in the Source form of any Derivative Works
113
+ that You distribute, all copyright, patent, trademark, and
114
+ attribution notices from the Source form of the Work,
115
+ excluding those notices that do not pertain to any part of
116
+ the Derivative Works; and
117
+
118
+ (d) If the Work includes a "NOTICE" text file as part of its
119
+ distribution, then any Derivative Works that You distribute must
120
+ include a readable copy of the attribution notices contained
121
+ within such NOTICE file, excluding those notices that do not
122
+ pertain to any part of the Derivative Works, in at least one
123
+ of the following places: within a NOTICE text file distributed
124
+ as part of the Derivative Works; within the Source form or
125
+ documentation, if provided along with the Derivative Works; or,
126
+ within a display generated by the Derivative Works, if and
127
+ wherever such third-party notices normally appear. The contents
128
+ of the NOTICE file are for informational purposes only and
129
+ do not modify the License. You may add Your own attribution
130
+ notices within Derivative Works that You distribute, alongside
131
+ or as an addendum to the NOTICE text from the Work, provided
132
+ that such additional attribution notices cannot be construed
133
+ as modifying the License.
134
+
135
+ You may add Your own copyright statement to Your modifications and
136
+ may provide additional or different license terms and conditions
137
+ for use, reproduction, or distribution of Your modifications, or
138
+ for any such Derivative Works as a whole, provided Your use,
139
+ reproduction, and distribution of the Work otherwise complies with
140
+ the conditions stated in this License.
141
+
142
+ 5. Submission of Contributions. Unless You explicitly state otherwise,
143
+ any Contribution intentionally submitted for inclusion in the Work
144
+ by You to the Licensor shall be under the terms and conditions of
145
+ this License, without any additional terms or conditions.
146
+ Notwithstanding the above, nothing herein shall supersede or modify
147
+ the terms of any separate license agreement you may have executed
148
+ with Licensor regarding such Contributions.
149
+
150
+ 6. Trademarks. This License does not grant permission to use the trade
151
+ names, trademarks, service marks, or product names of the Licensor,
152
+ except as required for reasonable and customary use in describing the
153
+ origin of the Work and reproducing the content of the NOTICE file.
154
+
155
+ 7. Disclaimer of Warranty. Unless required by applicable law or
156
+ agreed to in writing, Licensor provides the Work (and each
157
+ Contributor provides its Contributions) on an "AS IS" BASIS,
158
+ WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or
159
+ implied, including, without limitation, any warranties or conditions
160
+ of TITLE, NON-INFRINGEMENT, MERCHANTABILITY, or FITNESS FOR A
161
+ PARTICULAR PURPOSE. You are solely responsible for determining the
162
+ appropriateness of using or redistributing the Work and assume any
163
+ risks associated with Your exercise of permissions under this License.
164
+
165
+ 8. Limitation of Liability. In no event and under no legal theory,
166
+ whether in tort (including negligence), contract, or otherwise,
167
+ unless required by applicable law (such as deliberate and grossly
168
+ negligent acts) or agreed to in writing, shall any Contributor be
169
+ liable to You for damages, including any direct, indirect, special,
170
+ incidental, or consequential damages of any character arising as a
171
+ result of this License or out of the use or inability to use the
172
+ Work (including but not limited to damages for loss of goodwill,
173
+ work stoppage, computer failure or malfunction, or any and all
174
+ other commercial damages or losses), even if such Contributor
175
+ has been advised of the possibility of such damages.
176
+
177
+ 9. Accepting Warranty or Additional Liability. While redistributing
178
+ the Work or Derivative Works thereof, You may choose to offer,
179
+ and charge a fee for, acceptance of support, warranty, indemnity,
180
+ or other liability obligations and/or rights consistent with this
181
+ License. However, in accepting such obligations, You may act only
182
+ on Your own behalf and on Your sole responsibility, not on behalf
183
+ of any other Contributor, and only if You agree to indemnify,
184
+ defend, and hold each Contributor harmless for any liability
185
+ incurred by, or claims asserted against, such Contributor by reason
186
+ of your accepting any such warranty or additional liability.
187
+
188
+ END OF TERMS AND CONDITIONS
189
+
190
+ APPENDIX: How to apply the Apache License to your work.
191
+
192
+ To apply the Apache License to your work, attach the following
193
+ boilerplate notice, with the fields enclosed by brackets "{}"
194
+ replaced with your own identifying information. (Don't include
195
+ the brackets!) The text should be enclosed in the appropriate
196
+ comment syntax for the file format. We also recommend that a
197
+ file or class name and description of purpose be included on the
198
+ same "printed page" as the copyright notice for easier
199
+ identification within third-party archives.
200
+
201
+ Copyright {yyyy} {name of copyright owner}
202
+
203
+ Licensed under the Apache License, Version 2.0 (the "License");
204
+ you may not use this file except in compliance with the License.
205
+ You may obtain a copy of the License at
206
+
207
+ http://www.apache.org/licenses/LICENSE-2.0
208
+
209
+ Unless required by applicable law or agreed to in writing, software
210
+ distributed under the License is distributed on an "AS IS" BASIS,
211
+ WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
212
+ See the License for the specific language governing permissions and
213
+ limitations under the License.
214
+
215
+
216
+ ```
217
+
218
+ The shorter version of license text should be added as a comment to all Salesforce-authored source code and configuration files that support comments. This include file formats like HTML, CSS, JavaScript, XML, etc. which aren't directly code, but are still critical to your project code. Like:
219
+ ```
220
+ /*
221
+ * Copyright (c) 2023, Salesforce, Inc.
222
+ * SPDX-License-Identifier: Apache-2
223
+ *
224
+ * Licensed under the Apache License, Version 2.0 (the "License");
225
+ * you may not use this file except in compliance with the License.
226
+ * You may obtain a copy of the License at
227
+ *
228
+ * http://www.apache.org/licenses/LICENSE-2.0
229
+ *
230
+ * Unless required by applicable law or agreed to in writing, software
231
+ * distributed under the License is distributed on an "AS IS" BASIS,
232
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
233
+ * See the License for the specific language governing permissions and
234
+ * limitations under the License.
235
+ */
236
+ ```
237
+
238
+ Note that there are many tools that exist to do this sort of thing in an automated fashion, without having to manually edit every single file in your project. It is highly recommended that you research some of these tools for your particular language / build system.
239
+
240
+ For sample, demo, and example code, we recommend the [Unlicense](https://opensource.org/license/unlicense/) license. Create a `LICENSE.txt` file containing:
241
+ ```
242
+ This is free and unencumbered software released into the public domain.
243
+
244
+ Anyone is free to copy, modify, publish, use, compile, sell, or distribute this software, either in source code form or as a compiled binary, for any purpose, commercial or non-commercial, and by any means.
245
+
246
+ In jurisdictions that recognize copyright laws, the author or authors of this software dedicate any and all copyright interest in the software to the public domain. We make this dedication for the benefit of the public at large and to the detriment of our heirs and successors. We intend this dedication to be an overt act of relinquishment in perpetuity of all present and future rights to this software under copyright law.
247
+
248
+ THE SOFTWARE IS PROVIDED “AS IS”, WITHOUT WARRANTY OF ANY KIND, EXPRESS OR IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE AUTHORS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE SOFTWARE.
249
+ ```
250
+
251
+ No license header is required for samples, demos, and example code.
mcpuniverse.egg-info/PKG-INFO ADDED
@@ -0,0 +1,731 @@
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
1
+ Metadata-Version: 2.4
2
+ Name: mcpuniverse
3
+ Version: 1.1.3
4
+ Summary: A framework for developing and benchmarking AI agents using Model Context Protocol (MCP)
5
+ Author: Salesforce Research
6
+ License: Apache 2.0
7
+ Project-URL: Homepage, https://github.com/SalesforceAIResearch/MCP-Universe
8
+ Project-URL: Repository, https://github.com/SalesforceAIResearch/MCP-Universe
9
+ Keywords: AI,Agents,MCP,benchmarking,LLM,machine-learning
10
+ Classifier: Intended Audience :: Developers
11
+ Classifier: Programming Language :: Python :: 3
12
+ Classifier: Programming Language :: Python :: 3.10
13
+ Classifier: Programming Language :: Python :: 3.11
14
+ Classifier: Programming Language :: Python :: 3.12
15
+ Requires-Python: <4,>=3.10
16
+ Description-Content-Type: text/markdown
17
+ License-File: LICENSE.txt
18
+ Requires-Dist: requests==2.32.4
19
+ Requires-Dist: pydantic==2.11.7
20
+ Requires-Dist: pydantic[email]==2.11.7
21
+ Requires-Dist: mcp==1.13.1
22
+ Requires-Dist: httpx==0.28.1
23
+ Requires-Dist: click==8.1.8
24
+ Requires-Dist: jinja2==3.1.6
25
+ Requires-Dist: python-dotenv==1.0.1
26
+ Requires-Dist: anyio==4.9.0
27
+ Requires-Dist: openai==1.106.1
28
+ Requires-Dist: anthropic==0.49.0
29
+ Requires-Dist: mistralai==1.6.0
30
+ Requires-Dist: pyyaml==6.0.2
31
+ Requires-Dist: google-genai==1.16.1
32
+ Requires-Dist: redis==6.1.0
33
+ Requires-Dist: fastapi==0.115.12
34
+ Requires-Dist: uvicorn[standard]==0.34.0
35
+ Requires-Dist: bcrypt==4.3.0
36
+ Requires-Dist: pyseto==1.8.4
37
+ Requires-Dist: celery==5.5.3
38
+ Requires-Dist: xai-sdk==1.0.0
39
+ Requires-Dist: claude-code-sdk==0.0.20
40
+ Requires-Dist: openai-agents==0.2.11
41
+ Requires-Dist: wikipedia-api==0.8.1
42
+ Requires-Dist: mcp_server_fetch
43
+ Requires-Dist: google-auth==2.38.0
44
+ Requires-Dist: google-auth-oauthlib==1.2.1
45
+ Requires-Dist: google-api-python-client
46
+ Requires-Dist: mcp_server_calculator==0.1.1
47
+ Requires-Dist: yfinance==0.2.61
48
+ Requires-Dist: blender-mcp==1.1.3
49
+ Requires-Dist: playwright==1.52.0
50
+ Requires-Dist: mathutils==3.3.0
51
+ Requires-Dist: pytz==2024.2
52
+ Requires-Dist: tiktoken==0.11.0
53
+ Requires-Dist: kafka-python==2.2.15
54
+ Requires-Dist: pika==1.3.2
55
+ Requires-Dist: tenacity==9.1.2
56
+ Requires-Dist: loguru==0.7.3
57
+ Requires-Dist: aiohttp>=3.9.0
58
+ Requires-Dist: omegaconf>=2.3.0
59
+ Requires-Dist: beautifulsoup4>=4.12.0
60
+ Requires-Dist: pandas>=2.0.0
61
+ Requires-Dist: numpy>=1.24.0
62
+ Requires-Dist: notion-client==2.7.0
63
+ Requires-Dist: sqlalchemy[asyncio]==2.0.41
64
+ Provides-Extra: dev
65
+ Requires-Dist: pytest; extra == "dev"
66
+ Requires-Dist: pytest-asyncio; extra == "dev"
67
+ Requires-Dist: pytest-postgresql; extra == "dev"
68
+ Requires-Dist: pylint; extra == "dev"
69
+ Requires-Dist: pre-commit; extra == "dev"
70
+ Provides-Extra: web
71
+ Requires-Dist: uvicorn[standard]==0.34.0; extra == "web"
72
+ Requires-Dist: fastapi==0.115.12; extra == "web"
73
+ Requires-Dist: celery==5.5.3; extra == "web"
74
+ Requires-Dist: redis==6.1.0; extra == "web"
75
+ Requires-Dist: psycopg[binary]==3.2.9; extra == "web"
76
+ Requires-Dist: sqlalchemy[asyncio]==2.0.41; extra == "web"
77
+ Provides-Extra: dashboard
78
+ Requires-Dist: gradio>=5.42.0; extra == "dashboard"
79
+ Provides-Extra: deep-research
80
+ Requires-Dist: pillow==12.1.0; extra == "deep-research"
81
+ Requires-Dist: datasets==4.5.0; extra == "deep-research"
82
+ Requires-Dist: openpyxl==3.1.5; extra == "deep-research"
83
+ Requires-Dist: vertexai==1.71.1; extra == "deep-research"
84
+ Provides-Extra: vllm
85
+ Requires-Dist: vllm>=0.15.0; extra == "vllm"
86
+ Requires-Dist: ray>=2.0.0; extra == "vllm"
87
+ Requires-Dist: flash-attn>=2.7.0; extra == "vllm"
88
+ Requires-Dist: torch>=2.6.0; extra == "vllm"
89
+ Requires-Dist: nest_asyncio>=1.5.0; extra == "vllm"
90
+ Provides-Extra: sglang
91
+ Requires-Dist: sglang>=0.4.0; extra == "sglang"
92
+ Requires-Dist: ray>=2.0.0; extra == "sglang"
93
+ Requires-Dist: flash-attn>=2.7.0; extra == "sglang"
94
+ Requires-Dist: torch>=2.6.0; extra == "sglang"
95
+ Provides-Extra: rl
96
+ Requires-Dist: verl @ git+https://github.com/verl-project/verl.git@9433f8a8f2771256ea4f8f94e4401bcfe9703228 ; extra == "rl"
97
+ Requires-Dist: torch>=2.6.0; extra == "rl"
98
+ Requires-Dist: ray>=2.0.0; extra == "rl"
99
+ Requires-Dist: tensordict>=0.6.0; extra == "rl"
100
+ Requires-Dist: hydra-core>=1.3.0; extra == "rl"
101
+ Requires-Dist: tqdm>=4.60.0; extra == "rl"
102
+ Requires-Dist: transformers>=4.40.0; extra == "rl"
103
+ Requires-Dist: numpy>=1.24.0; extra == "rl"
104
+ Requires-Dist: accelerate>=1.0.0; extra == "rl"
105
+ Requires-Dist: peft>=0.10.0; extra == "rl"
106
+ Requires-Dist: wandb>=0.15.0; extra == "rl"
107
+ Requires-Dist: datasets>=4.0.0; extra == "rl"
108
+ Dynamic: license-file
109
+
110
+ # <img src="assets/icon.png" alt="MCP-Universe" width="23" height="23"> MCP-Universe
111
+
112
+ [![Paper](https://img.shields.io/badge/Paper-arXiv:2508.14704-B31B1B?style=for-the-badge&logo=arxiv&logoColor=white)](https://arxiv.org/abs/2508.14704)
113
+ [![Website](https://img.shields.io/badge/Website-Live-4285F4?style=for-the-badge&logo=googlechrome&logoColor=white)](https://mcp-universe.github.io/)
114
+ [![Leaderboard](https://img.shields.io/badge/Leaderboard-Results-FF6B35?style=for-the-badge&logo=chartdotjs&logoColor=white)](https://mcp-universe.github.io/#results)
115
+ [![Discord](https://img.shields.io/badge/Discord-Join_Community-5865F2?style=for-the-badge&logo=discord&logoColor=white)](https://discord.gg/t9tU77GF)
116
+
117
+ ### 🎉 Latest Updates
118
+
119
+ > **📊 [MCPMark Evaluation](#mcpmark-benchmark)** - MCP-Universe now supports evaluating the MCPMark tasks
120
+ >
121
+ > **🚀 [MCP+](#mcp-precision-context-management-for-mcp-agents)** - Agentic wrapper on MCP clients which reduce token costs by up to 75%
122
+ >
123
+ > **🔬 [Deep Research Agent](#deep-research-agent-wide--deep-wd-research)** - Scale the Width of Deep Research Agents with parallel tool calling, improving performance and efficiency
124
+
125
+ </div>
126
+
127
+ ---
128
+
129
+ ## What is MCP-Universe?
130
+
131
+ MCP-Universe is a comprehensive ecosystem for building, optimizing, and evaluating AI agents that interact with the Model Context Protocol (MCP). Beyond our industry-leading benchmark for real-world MCP server interactions, MCP-Universe provides production-ready tools for agent development including specialized research agents ([**Deep Research Agent**](#deep-research-agent-wide--deep-wd-research)), intelligent context management ([**MCP+**](#mcp-precision-context-management-for-mcp-agents)), and sophisticated orchestration workflows.
132
+
133
+ <div align="center">
134
+
135
+ ![MCP-Universe Introduction](assets/intro-mcp-universe.png)
136
+
137
+ </div>
138
+
139
+ **Benchmarking:** Unlike existing benchmarks that rely on overly simplistic tasks, MCP-Universe addresses critical gaps by evaluating LLMs in **real-world scenarios** through interaction with actual MCP servers, capturing real application challenges such as:
140
+
141
+ - 🎯 **Long-horizon reasoning** across multi-step tasks
142
+ - 🔧 **Large, unfamiliar tool spaces** with diverse MCP servers
143
+ - 🌍 **Real-world data sources** and live environments
144
+ - ⚡ **Dynamic evaluation** with time-sensitive ground truth
145
+
146
+
147
+ ## Table of Contents
148
+
149
+ - [What's New](#whats-new)
150
+ - [Architecture Overview](#architecture-overview)
151
+ - [Getting Started](#getting-started)
152
+ - [Prerequisites](#prerequisites)
153
+ - [Installation](#installation)
154
+ - [Quick Test](#quick-test)
155
+ - [Evaluating LLMs and Agents](#evaluating-llms-and-agents)
156
+ - [Prerequisites](#prerequisites-1)
157
+ - [Environment Configuration](#environment-configuration)
158
+ - [Benchmark Configuration](#benchmark-configuration)
159
+ - [Execution](#execution)
160
+ - [Save the running log](#save-the-running-log)
161
+ - [Save the benchmark result to a report](#save-the-benchmark-result-to-a-report)
162
+ - [Visualize the agent running information](#visualize-the-agent-running-information)
163
+ - [Creating Custom Benchmarks](#creating-custom-benchmarks)
164
+ - [Task definition](#task-definition)
165
+ - [Benchmark definition](#benchmark-definition)
166
+ - [Citation](#citation)
167
+
168
+ ## What's New
169
+
170
+ ### MCPMark Benchmark
171
+
172
+ **📊 Evaluate MCP Agents with MCPMark**
173
+
174
+ MCP-Universe now supports evaluating the **MCPMark** benchmark, enabling comprehensive testing and benchmarking of MCP agents. You can run MCPMark evaluations directly within the MCP-Universe framework to assess agent performance on MCP tasks.
175
+
176
+ **📚 Resources:**
177
+ - [How to run MCPMark](mcpuniverse/benchmark/configs/mcpmark/README.md#running-mcpmark-tasks)
178
+ - [Evaluation Scores](mcpuniverse/benchmark/configs/mcpmark/README.md#benchmark-results-alignment)
179
+
180
+ ---
181
+
182
+ ### MCP+: Precision Context Management for MCP Agents
183
+
184
+ **🚀 Reduce LLM Token Costs by up to 75% Without Sacrificing Quality**
185
+
186
+ MCP tools often return large, verbose outputs that waste your LLM's context window and cost money. **MCP+** wraps your MCP clients with intelligent post-processing that extracts only the relevant information before it reaches your LLM.
187
+
188
+ #### ✨ Key Features
189
+
190
+ - **💰 Massive Cost Reduction**: 50-75% token savings on tool outputs
191
+ - **⚡ Zero Code Changes**: Drop-in replacement for standard MCP clients
192
+
193
+
194
+ **📚 [Learn More at mcp-plus.github.io →](https://mcp-plus.github.io)**
195
+
196
+ </div>
197
+
198
+ ---
199
+
200
+ ### Deep Research Agent: Wide & Deep (W&D) Research
201
+
202
+ **🔬 Scale Research Width with Parallel Tool Calls**
203
+
204
+ **Feb 11, 2026** — We introduce **Wide & Deep (W&D) research agents** that scale *width* by making more parallel tool calls per turn. This approach improves accuracy on BrowseComp, HLE, and GAIA benchmarks while reducing turns, API cost, and wall-clock time. Our W&D agent with GPT-5-medium reaches **62.2%** on BrowseComp, outperforming GPT-5-high deep research (54.9%).
205
+
206
+ **📚 Resources:**
207
+ - [Paper](https://arxiv.org/pdf/2602.07359)
208
+ - [Website](https://xqlin98.github.io/wide-deep-research-agent/)
209
+ - [Code](mcpuniverse/benchmark/configs/deepresearch/README.md)
210
+
211
+ ---
212
+
213
+ ## Architecture Overview
214
+
215
+ The MCPUniverse architecture consists of the following key components:
216
+
217
+ - **Agents** (`mcpuniverse/agent/`): Base implementations for different agent types
218
+ - **Workflows** (`mcpuniverse/workflows/`): Orchestration and coordination layer
219
+ - **MCP Servers** (`mcpuniverse/mcp/`): Protocol management and external service integration
220
+ - **LLM Integration** (`mcpuniverse/llm/`): Multi-provider language model support
221
+ - **Benchmarking** (`mcpuniverse/benchmark/`): Evaluation and testing framework
222
+ - **Dashboard** (`mcpuniverse/dashboard/`): Visualization and monitoring interface
223
+
224
+ The diagram below illustrates the high-level view:
225
+
226
+ ```
227
+ ┌─────────────────────────────────────────────────────────────────┐
228
+ │ Application Layer │
229
+ ├─────────────────────────────────────────────────────────────────┤
230
+ │ Dashboard │ Web API │ Python Lib │ Benchmarks │
231
+ │ (Gradio) │ (FastAPI) │ │ │
232
+ └─────────────┬─────────────────┬────────────────┬────────────────┘
233
+ │ │ │
234
+ ┌─────────────▼─────────────────▼────────────────▼────────────────┐
235
+ │ Orchestration Layer │
236
+ ├─────────────────────────────────────────────────────────────────┤
237
+ │ Workflows │ Benchmark Runner │
238
+ │ (Chain, Router, etc.) │ (Evaluation Engine) │
239
+ └─────────────┬─────────────────┬────────────────┬────────────────┘
240
+ │ │ │
241
+ ┌─────────────▼─────────────────▼────────────────▼────────────────┐
242
+ │ Agent Layer │
243
+ ├─────────────────────────────────────────────────────────────────┤
244
+ │ BasicAgent │ ReActAgent │ FunctionCall │ Other │
245
+ │ │ │ Agent │ Agents │
246
+ └─────────────┬─────────────────┬────────────────┬────────────────┘
247
+ │ │ │
248
+ ┌─────────────▼─────────────────▼────────────────▼────────────────┐
249
+ │ Foundation Layer │
250
+ ├─────────────────────────────────────────────────────────────────┤
251
+ │ MCP Manager │ LLM Manager │ Memory Systems │ Tracers │
252
+ │ (Servers & │ (Multi-Model │ (RAM, Redis) │ (Logging) │
253
+ │ Clients) │ Support) │ │ │
254
+ └─────────────────┴─────────────────┴─────────────────┴───────────┘
255
+ ```
256
+
257
+ More information can be found [here](https://github.com/SalesforceAIResearch/MCP-Universe/blob/main/docs).
258
+
259
+ ## Getting Started
260
+
261
+ We follow
262
+ the [feature branch workflow](https://www.atlassian.com/git/tutorials/comparing-workflows/feature-branch-workflow)
263
+ in this repo for its simplicity. To ensure code quality, [PyLint](https://pylint.readthedocs.io/en/latest/)
264
+ is integrated into our CI to enforce Python coding standards.
265
+
266
+ ### Prerequisites
267
+
268
+ * **Python**: Requires version 3.10 or higher.
269
+ * **Docker**: Used for running Dockerized MCP servers.
270
+ * **PostgreSQL** (optional): Used for database storage and persistence.
271
+ * **Redis** (optional): Used for caching and memory management.
272
+
273
+ ### Installation
274
+
275
+ 1. **Clone the repository**
276
+ ```bash
277
+ git clone https://github.com/SalesforceAIResearch/MCP-Universe.git
278
+ cd MCP-Universe
279
+ ```
280
+
281
+ 2. **Create and activate virtual environment**
282
+ ```bash
283
+ python3 -m venv venv
284
+ source venv/bin/activate
285
+ ```
286
+
287
+ 3. **Install dependencies**
288
+ ```bash
289
+ pip install -r requirements.txt
290
+ pip install -r dev-requirements.txt
291
+ ```
292
+
293
+ 4. **Platform-specific requirements**
294
+
295
+ **Linux:**
296
+ ```bash
297
+ sudo apt-get install libpq-dev
298
+ ```
299
+
300
+ **macOS:**
301
+ ```bash
302
+ brew install postgresql
303
+ ```
304
+
305
+ 5. **Configure pre-commit hooks**
306
+ ```bash
307
+ pre-commit install
308
+ ```
309
+
310
+ 6. **Environment configuration**
311
+ ```bash
312
+ cp .env.example .env
313
+ # Edit .env with your API keys and configuration
314
+ ```
315
+
316
+ ### Quick Test
317
+
318
+ To run benchmarks, you first need to set environment variables:
319
+
320
+ 1. Copy the `.env.example` file to a new file named `.env`.
321
+ 2. In the `.env` file, set the required API keys for various services used by the agents,
322
+ such as `OPENAI_API_KEY` and `GOOGLE_MAPS_API_KEY`.
323
+
324
+ To execute a benchmark programmatically:
325
+
326
+ ```python
327
+ from mcpuniverse.tracer.collectors import MemoryCollector # You can also use SQLiteCollector
328
+ from mcpuniverse.benchmark.runner import BenchmarkRunner
329
+
330
+ async def test():
331
+ trace_collector = MemoryCollector()
332
+ # Choose a benchmark config file under the folder "mcpuniverse/benchmark/configs"
333
+ benchmark = BenchmarkRunner("dummy/benchmark_1.yaml")
334
+ # Run the specified benchmark
335
+ results = await benchmark.run(trace_collector=trace_collector)
336
+ # Get traces
337
+ trace_id = results[0].task_trace_ids["dummy/tasks/weather_1.json"]
338
+ trace_records = trace_collector.get(trace_id)
339
+ ```
340
+
341
+ ## Evaluating LLMs and Agents
342
+
343
+ This section provides comprehensive instructions for evaluating LLMs and AI agents using the MCP-Universe benchmark suite. The framework supports evaluation across multiple domains including web search, location navigation, browser automation, financial analysis, repository management, and 3D design.
344
+
345
+ ### Prerequisites
346
+
347
+ Before running benchmark evaluations, ensure you have completed the [Getting Started](#getting-started) section and have the following:
348
+
349
+ - Python: Version 3.10 or higher
350
+ - Docker: Installed and available in your environment
351
+ - All required dependencies installed via `pip install -r requirements.txt`
352
+ - Active virtual environment
353
+ - Appropriate API access for the services you intend to evaluate
354
+
355
+ ### Environment Configuration
356
+
357
+ #### 1. Initial Setup
358
+
359
+ Copy the environment template and configure your API credentials:
360
+
361
+ ```bash
362
+ cp .env.example .env
363
+ ```
364
+
365
+ #### 2. API Keys and Configuration
366
+
367
+ Configure the following environment variables in your `.env` file. The required keys depend on which benchmark domains you plan to evaluate:
368
+
369
+ ##### Core LLM Providers
370
+
371
+ | Environment Variable | Provider | Description | Required For |
372
+ |---------------------|----------|-------------|--------------|
373
+ | `OPENAI_API_KEY` | OpenAI | API key for GPT models (gpt-5, etc.) | All domains |
374
+ | `ANTHROPIC_API_KEY` | Anthropic | API key for Claude models | All domains |
375
+ | `GEMINI_API_KEY` | Google | API key for Gemini models | All domains |
376
+
377
+ > **Note**: You only need to configure the API key for the LLM provider you intend to use in your evaluation.
378
+
379
+ ##### Domain-Specific Services
380
+
381
+ | Environment Variable | Service | Description | Setup Instructions |
382
+ |---------------------|---------|-------------|-------------------|
383
+ | `SERP_API_KEY` | SerpAPI | Web search API for search benchmark evaluation | [Get API key](https://serpapi.com/) |
384
+ | `GOOGLE_MAPS_API_KEY` | Google Maps | Geolocation and mapping services | [Setup Guide](https://console.cloud.google.com/google/maps-apis/credentials) |
385
+ | `GITHUB_PERSONAL_ACCESS_TOKEN` | GitHub | Personal access token for repository operations | [Token Setup](https://docs.github.com/en/authentication/keeping-your-account-and-data-secure/managing-your-personal-access-tokens) |
386
+ | `GITHUB_PERSONAL_ACCOUNT_NAME` | GitHub | Your GitHub username | N/A |
387
+ | `NOTION_API_KEY` | Notion | Integration token for Notion workspace access | [Integration Setup](https://developers.notion.com/docs/authorization#obtaining-a-token) |
388
+ | `NOTION_ROOT_PAGE` | Notion | Root page ID for your Notion workspace | See configuration example below |
389
+
390
+ ##### System Paths
391
+
392
+ | Environment Variable | Description | Example |
393
+ |---------------------|-------------|---------|
394
+ | `BLENDER_APP_PATH` | Full path to Blender executable (we used v4.4.0) | `/Applications/Blender.app/Contents/MacOS/Blender` |
395
+ | `MCPUniverse_DIR` | Absolute path to your MCP-Universe repository | `/Users/username/MCP-Universe` |
396
+
397
+ ##### Configuration Examples
398
+
399
+ **Notion Root Page ID:**
400
+ If your Notion page URL is:
401
+ ```
402
+ https://www.notion.so/your_workspace/MCP-Evaluation-1dd6d96e12345678901234567eaf9eff
403
+ ```
404
+ Set `NOTION_ROOT_PAGE=MCP-Evaluation-1dd6d96e12345678901234567eaf9eff`
405
+
406
+ **Blender Installation:**
407
+ 1. Download Blender v4.4.0 from [blender.org](https://www.blender.org/)
408
+ 2. Install our modified Blender MCP server following the [installation guide](docs/blender-setup.md)
409
+ 3. Set the path to the Blender executable
410
+
411
+ ##### ⚠️ Security Recommendations
412
+
413
+ > **🔒 IMPORTANT SECURITY NOTICE**
414
+ >
415
+ > Please read and follow these security guidelines carefully before running benchmarks:
416
+
417
+ - **🚨 GitHub Integration**: **CRITICAL** - We strongly recommend using a dedicated test GitHub account for benchmark evaluation. The AI agent will perform real operations on GitHub repositories, which could potentially modify or damage your personal repositories.
418
+
419
+ - **🔐 API Key Management**:
420
+ - Store API keys securely and never commit them to version control
421
+ - Use environment variables or secure key management systems
422
+ - Regularly rotate your API keys for enhanced security
423
+
424
+ - **🛡️ Access Permissions**:
425
+ - Grant minimal necessary permissions for each service integration
426
+ - Review and limit API key scopes to only required operations
427
+ - Monitor API usage and set appropriate rate limits
428
+
429
+ - **⚡ Blender Operations**: The 3D design benchmarks will execute Blender commands that may modify or create files on your system. Ensure you have adequate backups and run in an isolated environment if necessary.
430
+
431
+ ### Benchmark Configuration
432
+
433
+ #### Domain-Specific Configuration Files
434
+
435
+ Each benchmark domain has a dedicated YAML configuration file located in `mcpuniverse/benchmark/configs/test/`. To evaluate your LLM/agent, modify the appropriate configuration file:
436
+
437
+ | Domain | Configuration File | Description |
438
+ |--------|-------------------|-------------|
439
+ | Web Search | `web_search.yaml` | Search engine and information retrieval tasks |
440
+ | Location Navigation | `location_navigation.yaml` | Geographic and mapping-related queries |
441
+ | Browser Automation | `browser_automation.yaml` | Web interaction and automation scenarios |
442
+ | Financial Analysis | `financial_analysis.yaml` | Market data analysis and financial computations |
443
+ | Repository Management | `repository_management.yaml` | Git operations and code repository tasks |
444
+ | 3D Design | `3d_design.yaml` | Blender-based 3D modeling and design tasks |
445
+
446
+ #### LLM Model Configuration
447
+
448
+ In each configuration file, update the LLM specification to match your target model:
449
+
450
+ ```yaml
451
+ kind: llm
452
+ spec:
453
+ name: llm-1
454
+ type: openai # or anthropic, google, etc.
455
+ config:
456
+ model_name: gpt-4o # Replace with your target model
457
+ ```
458
+
459
+ ### Execution
460
+
461
+ #### Running Individual Benchmarks
462
+
463
+ Execute specific domain benchmarks using the following commands:
464
+
465
+ ```bash
466
+ # Set Python path and run individual benchmarks
467
+ export PYTHONPATH=.
468
+
469
+ # Location Navigation
470
+ python tests/benchmark/mcpuniverse/test_benchmark_location_navigation.py
471
+
472
+ # Browser Automation
473
+ python tests/benchmark/mcpuniverse/test_benchmark_browser_automation.py
474
+
475
+ # Financial Analysis
476
+ python tests/benchmark/mcpuniverse/test_benchmark_financial_analysis.py
477
+
478
+ # Repository Management
479
+ python tests/benchmark/mcpuniverse/test_benchmark_repository_management.py
480
+
481
+ # Web Search
482
+ python tests/benchmark/mcpuniverse/test_benchmark_web_search.py
483
+
484
+ # 3D Design
485
+ python tests/benchmark/mcpuniverse/test_benchmark_3d_design.py
486
+ ```
487
+
488
+ #### Batch Execution
489
+
490
+ For comprehensive evaluation across all domains:
491
+
492
+ ```bash
493
+ #!/bin/bash
494
+ export PYTHONPATH=.
495
+
496
+ domains=("location_navigation" "browser_automation" "financial_analysis"
497
+ "repository_management" "web_search" "3d_design")
498
+
499
+ for domain in "${domains[@]}"; do
500
+ echo "Running benchmark: $domain"
501
+ python "tests/benchmark/mcpuniverse/test_benchmark_${domain}.py"
502
+ echo "Completed: $domain"
503
+ done
504
+ ```
505
+
506
+ ### Save the running log
507
+
508
+ If you want to save the running log, you can pass the `trace_collector` to the benchmark run function:
509
+
510
+ ```python
511
+ from mcpuniverse.tracer.collectors import FileCollector
512
+
513
+ trace_collector = FileCollector(log_file="log/location_navigation.log")
514
+ benchmark_results = await benchmark.run(trace_collector=trace_collector)
515
+ ```
516
+
517
+ ### Save the benchmark result to a report
518
+
519
+ If you want to save a report of the benchmark result, you can use `BenchmarkReport` to dump a report:
520
+
521
+ ```python
522
+ from mcpuniverse.benchmark.report import BenchmarkReport
523
+
524
+ report = BenchmarkReport(benchmark, trace_collector=trace_collector)
525
+ report.dump()
526
+ ```
527
+
528
+ ### Visualize the agent running information
529
+
530
+ To run the benchmark with intermediate results and see real-time progress, pass `callbacks=get_vprint_callbacks()` to the run function:
531
+
532
+ ```python
533
+ from mcpuniverse.callbacks.handlers.vprint import get_vprint_callbacks
534
+
535
+ benchmark_results = await benchmark.run(
536
+ trace_collector=trace_collector,
537
+ callbacks=get_vprint_callbacks()
538
+ )
539
+ ```
540
+
541
+ This will print out the intermediate results as the benchmark runs.
542
+
543
+
544
+ For further details, refer to the in-code documentation or existing configuration samples in the repository.
545
+
546
+ ## Creating Custom Benchmarks
547
+
548
+ A benchmark is defined by three main configuration elements: the task definition,
549
+ agent/workflow definition, and the benchmark configuration itself. Below is an example
550
+ using a simple "weather forecasting" task.
551
+
552
+ ### Task definition
553
+
554
+ The task definition is provided in JSON format, for example:
555
+
556
+ ```json
557
+ {
558
+ "category": "general",
559
+ "question": "What's the weather in San Francisco now?",
560
+ "mcp_servers": [
561
+ {
562
+ "name": "weather"
563
+ }
564
+ ],
565
+ "output_format": {
566
+ "city": "<City>",
567
+ "weather": "<Weather forecast results>"
568
+ },
569
+ "evaluators": [
570
+ {
571
+ "func": "json -> get(city)",
572
+ "op": "=",
573
+ "value": "San Francisco"
574
+ }
575
+ ]
576
+ }
577
+ ```
578
+
579
+ Field descriptions:
580
+
581
+ 1. **category**: The task category, e.g., "general", "google-maps", etc. You can set any value for this property.
582
+ 2. **question**: The main question you want to ask in this task. This is treated as a user message.
583
+ 3. **mcp_servers**: A list of MCP servers that are supported in this framework.
584
+ 4. **output_format**: The desired output format of agent responses.
585
+ 5. **evaluators**: A list of tests to evaluate. For each test/evaluator, it has three attributes: "func" indicates
586
+ how to extract values from the agent response, "op" is the comparison operator, and "value" is the ground-truth
587
+ value.
588
+ It will evaluate **op(func(...), value, op_args...)**. "op" can be "=", "<", ">" or other customized operators.
589
+
590
+ In "evaluators", you need to write a rule ("func" attribute) showing how to extract values for testing. In the example
591
+ above, "json -> get(city)" will first do JSON decoding and then extract the value of key "city". There are several
592
+ predefined funcs in this repo:
593
+
594
+ 1. **json**: Perform JSON decoding.
595
+ 2. **get**: Get the value of a key.
596
+ 3. **len**: Get the length of a list.
597
+ 4. **foreach**: Do a FOR-EACH loop.
598
+
599
+ For example, let's define
600
+
601
+ ```python
602
+ data = {"x": [{"y": [1]}, {"y": [1, 1]}, {"y": [1, 2, 3, 4]}]}
603
+ ```
604
+
605
+ Then `get(x) -> foreach -> get(y) -> len` will do the following:
606
+
607
+ 1. Get the value of "x": `[{"y": [1]}, {"y": [1, 1]}, {"y": [1, 2, 3, 4]}]`.
608
+ 2. Do a foreach loop and get the value of "y": `[[1], [1, 1], [1, 2, 3, 4]]`.
609
+ 3. Get the length of each list: `[1, 2, 4]`.
610
+
611
+ If these predefined functions are not enough, you can implement custom ones.
612
+ For more details, please check
613
+ this [doc](https://github.com/SalesforceAIResearch/MCP-Universe/blob/main/docs/custom-evaluators-guide.md).
614
+
615
+ ### Benchmark definition
616
+
617
+ Define agent(s) and benchmark in a YAML file. Here’s a simple weather forecast benchmark:
618
+
619
+ ```yaml
620
+ kind: llm
621
+ spec:
622
+ name: llm-1
623
+ type: openai
624
+ config:
625
+ model_name: gpt-4o
626
+
627
+ ---
628
+ kind: agent
629
+ spec:
630
+ name: ReAct-agent
631
+ type: react
632
+ config:
633
+ llm: llm-1
634
+ instruction: You are an agent for weather forecasting.
635
+ servers:
636
+ - name: weather
637
+
638
+ ---
639
+ kind: benchmark
640
+ spec:
641
+ description: Test the agent for weather forecasting
642
+ agent: ReAct-agent
643
+ tasks:
644
+ - dummy/tasks/weather.json
645
+ ```
646
+
647
+ The benchmark definition mainly contains two parts: the agent definition and the benchmark configuration. The benchmark configuration is simple—you just need to specify the agent to use (by the defined agent name) and a list of tasks to evaluate. Each task entry is the task config file
648
+ path. It can be a full file path or a partial file path. If it is a partial file path (like "dummy/tasks/weather.json"),
649
+ it should be put in the
650
+ folder [mcpuniverse/benchmark/configs](https://github.com/SalesforceAIResearch/MCP-Universe/tree/main/mcpuniverse/benchmark/configs)
651
+ in this repo.
652
+
653
+ This framework offers a flexible way to define both simple agents (such as ReAct) and more complex, multi-step agent
654
+ workflows.
655
+
656
+ 1. **Specify LLMs:** Begin by declaring the large language models (LLMs) you want the agents to use. Each LLM component
657
+ must be assigned a unique name (e.g., `"llm-1"`). These names serve as identifiers that the framework uses to connect
658
+ the different components together.
659
+ 2. **Define an agent:** Next, define an agent by providing its name and selecting an agent class. Agent classes are
660
+ available in
661
+ the [mcpuniverse.agent](https://github.com/SalesforceAIResearch/MCP-Universe/tree/main/mcpuniverse/agent) package.
662
+ Commonly used classes include `"basic"`, `"function-call"`, and `"react"`. Within the agent specification (
663
+ `spec.config`), you must also indicate which LLM instance the agent should use by setting the `"llm"` field.
664
+ 3. **Create complex workflows:** Beyond simple agents, the framework supports the definition of sophisticated,
665
+ orchestrated workflows where multiple agents interact or collaborate to solve more complex tasks.
666
+
667
+ For example:
668
+
669
+ ```yaml
670
+ kind: llm
671
+ spec:
672
+ name: llm-1
673
+ type: openai
674
+ config:
675
+ model_name: gpt-4o
676
+
677
+ ---
678
+ kind: agent
679
+ spec:
680
+ name: basic-agent
681
+ type: basic
682
+ config:
683
+ llm: llm-1
684
+ instruction: Return the latitude and the longitude of a place.
685
+
686
+ ---
687
+ kind: agent
688
+ spec:
689
+ name: function-call-agent
690
+ type: function-call
691
+ config:
692
+ llm: llm-1
693
+ instruction: You are an agent for weather forecast. Please return the weather today at the given latitude and longitude.
694
+ servers:
695
+ - name: weather
696
+
697
+ ---
698
+ kind: workflow
699
+ spec:
700
+ name: orchestrator-workflow
701
+ type: orchestrator
702
+ config:
703
+ llm: llm-1
704
+ agents:
705
+ - basic-agent
706
+ - function-call-agent
707
+
708
+ ---
709
+ kind: benchmark
710
+ spec:
711
+ description: Test the agent for weather forecasting
712
+ agent: orchestrator-workflow
713
+ tasks:
714
+ - dummy/tasks/weather.json
715
+ ```
716
+
717
+ ## Citation
718
+
719
+ If you use MCP-Universe in your research, please cite our paper:
720
+
721
+ ```bibtex
722
+ @misc{mcpuniverse,
723
+ title={MCP-Universe: Benchmarking Large Language Models with Real-World Model Context Protocol Servers},
724
+ author={Ziyang Luo and Zhiqi Shen and Wenzhuo Yang and Zirui Zhao and Prathyusha Jwalapuram and Amrita Saha and Doyen Sahoo and Silvio Savarese and Caiming Xiong and Junnan Li},
725
+ year={2025},
726
+ eprint={2508.14704},
727
+ archivePrefix={arXiv},
728
+ primaryClass={cs.AI},
729
+ url={https://arxiv.org/abs/2508.14704},
730
+ }
731
+ ```
mcpuniverse.egg-info/SOURCES.txt ADDED
@@ -0,0 +1,437 @@
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
1
+ LICENSE.txt
2
+ README.md
3
+ pyproject.toml
4
+ mcpuniverse/__init__.py
5
+ mcpuniverse.egg-info/PKG-INFO
6
+ mcpuniverse.egg-info/SOURCES.txt
7
+ mcpuniverse.egg-info/dependency_links.txt
8
+ mcpuniverse.egg-info/entry_points.txt
9
+ mcpuniverse.egg-info/requires.txt
10
+ mcpuniverse.egg-info/top_level.txt
11
+ mcpuniverse/agent/__init__.py
12
+ mcpuniverse/agent/base.py
13
+ mcpuniverse/agent/basic.py
14
+ mcpuniverse/agent/claude_code.py
15
+ mcpuniverse/agent/depreciated_function_call.py
16
+ mcpuniverse/agent/explore_and_exploit.py
17
+ mcpuniverse/agent/function_call.py
18
+ mcpuniverse/agent/function_call_wide.py
19
+ mcpuniverse/agent/function_call_wide_claude.py
20
+ mcpuniverse/agent/harmony_agent.py
21
+ mcpuniverse/agent/manager.py
22
+ mcpuniverse/agent/openai_agent_sdk.py
23
+ mcpuniverse/agent/react.py
24
+ mcpuniverse/agent/react_train_agent.py
25
+ mcpuniverse/agent/reflection.py
26
+ mcpuniverse/agent/types.py
27
+ mcpuniverse/agent/utils.py
28
+ mcpuniverse/agent/workflow.py
29
+ mcpuniverse/agent/configs/explore_and_exploit_prompt.j2
30
+ mcpuniverse/agent/configs/function_call_prompt.j2
31
+ mcpuniverse/agent/configs/function_call_prompt_scheduler.j2
32
+ mcpuniverse/agent/configs/openai_agent_sdk_prompt.j2
33
+ mcpuniverse/agent/configs/react_prompt.j2
34
+ mcpuniverse/agent/configs/reflection_prompt.j2
35
+ mcpuniverse/agent/configs/system_prompt.j2
36
+ mcpuniverse/agent/configs/tools_prompt.j2
37
+ mcpuniverse/agent/memory/__init__.py
38
+ mcpuniverse/agent/memory/base.py
39
+ mcpuniverse/agent/memory/short_term/__init__.py
40
+ mcpuniverse/agent/memory/short_term/ram.py
41
+ mcpuniverse/agent/memory/short_term/redis.py
42
+ mcpuniverse/app/__init__.py
43
+ mcpuniverse/app/main.py
44
+ mcpuniverse/app/server.py
45
+ mcpuniverse/app/api/__init__.py
46
+ mcpuniverse/app/api/benchmark.py
47
+ mcpuniverse/app/api/chat.py
48
+ mcpuniverse/app/api/job.py
49
+ mcpuniverse/app/api/middleware.py
50
+ mcpuniverse/app/api/project.py
51
+ mcpuniverse/app/api/task.py
52
+ mcpuniverse/app/api/user.py
53
+ mcpuniverse/app/core/__init__.py
54
+ mcpuniverse/app/core/engine.py
55
+ mcpuniverse/app/db/__init__.py
56
+ mcpuniverse/app/db/database.py
57
+ mcpuniverse/app/db/migration.py
58
+ mcpuniverse/app/db/migration/000001_init.down.sql
59
+ mcpuniverse/app/db/migration/000001_init.up.sql
60
+ mcpuniverse/app/db/sqlc/__init__.py
61
+ mcpuniverse/app/db/sqlc/benchmark.py
62
+ mcpuniverse/app/db/sqlc/benchmark_job.py
63
+ mcpuniverse/app/db/sqlc/models.py
64
+ mcpuniverse/app/db/sqlc/models_sqlalchemy.py
65
+ mcpuniverse/app/db/sqlc/project.py
66
+ mcpuniverse/app/db/sqlc/released_benchmark.py
67
+ mcpuniverse/app/db/sqlc/released_project.py
68
+ mcpuniverse/app/db/sqlc/released_task.py
69
+ mcpuniverse/app/db/sqlc/task.py
70
+ mcpuniverse/app/db/sqlc/user.py
71
+ mcpuniverse/app/tasks/__init__.py
72
+ mcpuniverse/app/tasks/benchmark.py
73
+ mcpuniverse/app/tasks/celery_config.py
74
+ mcpuniverse/app/tasks/echo.py
75
+ mcpuniverse/app/tasks/worker.py
76
+ mcpuniverse/app/utils/__init__.py
77
+ mcpuniverse/app/utils/limiter.py
78
+ mcpuniverse/app/utils/redis.py
79
+ mcpuniverse/app/utils/token.py
80
+ mcpuniverse/benchmark/__init__.py
81
+ mcpuniverse/benchmark/cleanups.py
82
+ mcpuniverse/benchmark/report.py
83
+ mcpuniverse/benchmark/runner.py
84
+ mcpuniverse/benchmark/task.py
85
+ mcpuniverse/benchmark/configs/deepresearch/data_utils.py
86
+ mcpuniverse/benchmark/configs/deepresearch/prepare_deep_research_data.py
87
+ mcpuniverse/benchmark/configs/mcpmark/mcpmark_utils.py
88
+ mcpuniverse/benchmark/configs/mcpmark/prepares.py
89
+ mcpuniverse/callbacks/__init__.py
90
+ mcpuniverse/callbacks/base.py
91
+ mcpuniverse/callbacks/handlers/__init__.py
92
+ mcpuniverse/callbacks/handlers/memory.py
93
+ mcpuniverse/callbacks/handlers/redis.py
94
+ mcpuniverse/callbacks/handlers/sqlite.py
95
+ mcpuniverse/callbacks/handlers/vprint.py
96
+ mcpuniverse/common/__init__.py
97
+ mcpuniverse/common/config.py
98
+ mcpuniverse/common/context.py
99
+ mcpuniverse/common/logger.py
100
+ mcpuniverse/common/misc.py
101
+ mcpuniverse/dashboard/__init__.py
102
+ mcpuniverse/dashboard/app.py
103
+ mcpuniverse/dashboard/manager.py
104
+ mcpuniverse/dashboard/pages/__init__.py
105
+ mcpuniverse/dashboard/pages/agent.py
106
+ mcpuniverse/dashboard/pages/benchmark.py
107
+ mcpuniverse/dashboard/pages/chatbot.py
108
+ mcpuniverse/dashboard/pages/utils.py
109
+ mcpuniverse/dashboard/static/styles.css
110
+ mcpuniverse/dashboard/templates/index.html
111
+ mcpuniverse/evaluator/__init__.py
112
+ mcpuniverse/evaluator/evaluator.py
113
+ mcpuniverse/evaluator/functions.py
114
+ mcpuniverse/evaluator/blender/__init__.py
115
+ mcpuniverse/evaluator/blender/functions.py
116
+ mcpuniverse/evaluator/blender/check_functions/tid_10_check.py
117
+ mcpuniverse/evaluator/blender/check_functions/tid_11_check.py
118
+ mcpuniverse/evaluator/blender/check_functions/tid_12_check.py
119
+ mcpuniverse/evaluator/blender/check_functions/tid_13_check.py
120
+ mcpuniverse/evaluator/blender/check_functions/tid_14_check.py
121
+ mcpuniverse/evaluator/blender/check_functions/tid_15_check.py
122
+ mcpuniverse/evaluator/blender/check_functions/tid_16_check.py
123
+ mcpuniverse/evaluator/blender/check_functions/tid_17_check.py
124
+ mcpuniverse/evaluator/blender/check_functions/tid_19_check.py
125
+ mcpuniverse/evaluator/blender/check_functions/tid_1_check.py
126
+ mcpuniverse/evaluator/blender/check_functions/tid_20_check.py
127
+ mcpuniverse/evaluator/blender/check_functions/tid_2_check.py
128
+ mcpuniverse/evaluator/blender/check_functions/tid_3_check.py
129
+ mcpuniverse/evaluator/blender/check_functions/tid_4_check.py
130
+ mcpuniverse/evaluator/blender/check_functions/tid_5_check.py
131
+ mcpuniverse/evaluator/blender/check_functions/tid_6_check.py
132
+ mcpuniverse/evaluator/blender/check_functions/tid_7_check.py
133
+ mcpuniverse/evaluator/blender/check_functions/tid_8_check.py
134
+ mcpuniverse/evaluator/blender/check_functions/tid_9_check.py
135
+ mcpuniverse/evaluator/deepresearch/__init__.py
136
+ mcpuniverse/evaluator/deepresearch/functions.py
137
+ mcpuniverse/evaluator/github/__init__.py
138
+ mcpuniverse/evaluator/github/functions.py
139
+ mcpuniverse/evaluator/google_maps/__init__.py
140
+ mcpuniverse/evaluator/google_maps/functions.py
141
+ mcpuniverse/evaluator/google_search/__init__.py
142
+ mcpuniverse/evaluator/google_search/functions.py
143
+ mcpuniverse/evaluator/mcpmark/__init__.py
144
+ mcpuniverse/evaluator/mcpmark/filesystem_functions.py
145
+ mcpuniverse/evaluator/mcpmark/github_functions.py
146
+ mcpuniverse/evaluator/mcpmark/notion_functions.py
147
+ mcpuniverse/evaluator/mcpmark/playwright_functions.py
148
+ mcpuniverse/evaluator/mcpmark/postgres_functions.py
149
+ mcpuniverse/evaluator/mcpmark/filesystem/desktop/music_report/verify.py
150
+ mcpuniverse/evaluator/mcpmark/filesystem/desktop/project_management/verify.py
151
+ mcpuniverse/evaluator/mcpmark/filesystem/desktop/timeline_extraction/verify.py
152
+ mcpuniverse/evaluator/mcpmark/filesystem/desktop_template/budget_computation/verify.py
153
+ mcpuniverse/evaluator/mcpmark/filesystem/desktop_template/contact_information/verify.py
154
+ mcpuniverse/evaluator/mcpmark/filesystem/desktop_template/file_arrangement/verify.py
155
+ mcpuniverse/evaluator/mcpmark/filesystem/file_context/duplicates_searching/verify.py
156
+ mcpuniverse/evaluator/mcpmark/filesystem/file_context/file_merging/verify.py
157
+ mcpuniverse/evaluator/mcpmark/filesystem/file_context/file_splitting/verify.py
158
+ mcpuniverse/evaluator/mcpmark/filesystem/file_context/pattern_matching/verify.py
159
+ mcpuniverse/evaluator/mcpmark/filesystem/file_context/uppercase/verify.py
160
+ mcpuniverse/evaluator/mcpmark/filesystem/file_property/size_classification/verify.py
161
+ mcpuniverse/evaluator/mcpmark/filesystem/file_property/time_classification/verify.py
162
+ mcpuniverse/evaluator/mcpmark/filesystem/folder_structure/structure_analysis/verify.py
163
+ mcpuniverse/evaluator/mcpmark/filesystem/folder_structure/structure_mirror/verify.py
164
+ mcpuniverse/evaluator/mcpmark/filesystem/legal_document/dispute_review/verify.py
165
+ mcpuniverse/evaluator/mcpmark/filesystem/legal_document/individual_comments/verify.py
166
+ mcpuniverse/evaluator/mcpmark/filesystem/legal_document/solution_tracing/verify.py
167
+ mcpuniverse/evaluator/mcpmark/filesystem/papers/author_folders/verify.py
168
+ mcpuniverse/evaluator/mcpmark/filesystem/papers/find_math_paper/verify.py
169
+ mcpuniverse/evaluator/mcpmark/filesystem/papers/organize_legacy_papers/verify.py
170
+ mcpuniverse/evaluator/mcpmark/filesystem/student_database/duplicate_name/verify.py
171
+ mcpuniverse/evaluator/mcpmark/filesystem/student_database/english_talent/verify.py
172
+ mcpuniverse/evaluator/mcpmark/filesystem/student_database/gradebased_score/verify.py
173
+ mcpuniverse/evaluator/mcpmark/filesystem/threestudio/code_locating/verify.py
174
+ mcpuniverse/evaluator/mcpmark/filesystem/threestudio/output_analysis/verify.py
175
+ mcpuniverse/evaluator/mcpmark/filesystem/threestudio/requirements_completion/verify.py
176
+ mcpuniverse/evaluator/mcpmark/filesystem/votenet/dataset_comparison/verify.py
177
+ mcpuniverse/evaluator/mcpmark/filesystem/votenet/debugging/verify.py
178
+ mcpuniverse/evaluator/mcpmark/filesystem/votenet/requirements_writing/verify.py
179
+ mcpuniverse/evaluator/mcpmark/github/build_your_own_x/find_commit_date/verify.py
180
+ mcpuniverse/evaluator/mcpmark/github/build_your_own_x/find_rag_commit/verify.py
181
+ mcpuniverse/evaluator/mcpmark/github/claude_code/__init__.py
182
+ mcpuniverse/evaluator/mcpmark/github/claude_code/automated_changelog_generation/verify.py
183
+ mcpuniverse/evaluator/mcpmark/github/claude_code/claude_collaboration_analysis/verify.py
184
+ mcpuniverse/evaluator/mcpmark/github/claude_code/critical_issue_hotfix_workflow/verify.py
185
+ mcpuniverse/evaluator/mcpmark/github/claude_code/feature_commit_tracking/verify.py
186
+ mcpuniverse/evaluator/mcpmark/github/claude_code/label_color_standardization/verify.py
187
+ mcpuniverse/evaluator/mcpmark/github/easyr1/advanced_branch_strategy/verify.py
188
+ mcpuniverse/evaluator/mcpmark/github/easyr1/config_parameter_audit/verify.py
189
+ mcpuniverse/evaluator/mcpmark/github/easyr1/performance_regression_investigation/verify.py
190
+ mcpuniverse/evaluator/mcpmark/github/easyr1/qwen3_issue_management/verify.py
191
+ mcpuniverse/evaluator/mcpmark/github/harmony/fix_conflict/verify.py
192
+ mcpuniverse/evaluator/mcpmark/github/harmony/issue_pr_commit_workflow/verify.py
193
+ mcpuniverse/evaluator/mcpmark/github/harmony/issue_tagging_pr_closure/verify.py
194
+ mcpuniverse/evaluator/mcpmark/github/harmony/multi_branch_commit_aggregation/verify.py
195
+ mcpuniverse/evaluator/mcpmark/github/harmony/release_management_workflow/verify.py
196
+ mcpuniverse/evaluator/mcpmark/github/mcpmark_cicd/__init__.py
197
+ mcpuniverse/evaluator/mcpmark/github/mcpmark_cicd/deployment_status_workflow/verify.py
198
+ mcpuniverse/evaluator/mcpmark/github/mcpmark_cicd/issue_management_workflow/verify.py
199
+ mcpuniverse/evaluator/mcpmark/github/mcpmark_cicd/linting_ci_workflow/verify.py
200
+ mcpuniverse/evaluator/mcpmark/github/mcpmark_cicd/pr_automation_workflow/verify.py
201
+ mcpuniverse/evaluator/mcpmark/github/missing_semester/__init__.py
202
+ mcpuniverse/evaluator/mcpmark/github/missing_semester/assign_contributor_labels/verify.py
203
+ mcpuniverse/evaluator/mcpmark/github/missing_semester/find_legacy_name/verify.py
204
+ mcpuniverse/evaluator/mcpmark/github/missing_semester/find_salient_file/verify.py
205
+ mcpuniverse/evaluator/mcpmark/notion/company_in_a_box/employee_onboarding/verify.py
206
+ mcpuniverse/evaluator/mcpmark/notion/company_in_a_box/goals_restructure/verify.py
207
+ mcpuniverse/evaluator/mcpmark/notion/company_in_a_box/quarterly_review_dashboard/verify.py
208
+ mcpuniverse/evaluator/mcpmark/notion/computer_science_student_dashboard/code_snippets_go/verify.py
209
+ mcpuniverse/evaluator/mcpmark/notion/computer_science_student_dashboard/courses_internships_relation/verify.py
210
+ mcpuniverse/evaluator/mcpmark/notion/computer_science_student_dashboard/study_session_tracker/verify.py
211
+ mcpuniverse/evaluator/mcpmark/notion/it_trouble_shooting_hub/asset_retirement_migration/verify.py
212
+ mcpuniverse/evaluator/mcpmark/notion/it_trouble_shooting_hub/security_audit_ticket/verify.py
213
+ mcpuniverse/evaluator/mcpmark/notion/it_trouble_shooting_hub/verification_expired_update/verify.py
214
+ mcpuniverse/evaluator/mcpmark/notion/japan_travel_planner/daily_itinerary_overview/verify.py
215
+ mcpuniverse/evaluator/mcpmark/notion/japan_travel_planner/packing_progress_summary/verify.py
216
+ mcpuniverse/evaluator/mcpmark/notion/japan_travel_planner/remove_osaka_itinerary/verify.py
217
+ mcpuniverse/evaluator/mcpmark/notion/japan_travel_planner/restaurant_expenses_sync/verify.py
218
+ mcpuniverse/evaluator/mcpmark/notion/online_resume/layout_adjustment/verify.py
219
+ mcpuniverse/evaluator/mcpmark/notion/online_resume/projects_section_update/verify.py
220
+ mcpuniverse/evaluator/mcpmark/notion/online_resume/skills_development_tracker/verify.py
221
+ mcpuniverse/evaluator/mcpmark/notion/online_resume/work_history_addition/verify.py
222
+ mcpuniverse/evaluator/mcpmark/notion/python_roadmap/expert_level_lessons/verify.py
223
+ mcpuniverse/evaluator/mcpmark/notion/python_roadmap/learning_metrics_dashboard/verify.py
224
+ mcpuniverse/evaluator/mcpmark/notion/self_assessment/faq_column_layout/verify.py
225
+ mcpuniverse/evaluator/mcpmark/notion/self_assessment/hyperfocus_analysis_report/verify.py
226
+ mcpuniverse/evaluator/mcpmark/notion/self_assessment/numbered_list_emojis/verify.py
227
+ mcpuniverse/evaluator/mcpmark/notion/standard_operating_procedure/deployment_process_sop/verify.py
228
+ mcpuniverse/evaluator/mcpmark/notion/standard_operating_procedure/section_organization/verify.py
229
+ mcpuniverse/evaluator/mcpmark/notion/team_projects/priority_tasks_table/verify.py
230
+ mcpuniverse/evaluator/mcpmark/notion/team_projects/swap_tasks/verify.py
231
+ mcpuniverse/evaluator/mcpmark/notion/toronto_guide/change_color/verify.py
232
+ mcpuniverse/evaluator/mcpmark/notion/toronto_guide/weekend_adventure_planner/verify.py
233
+ mcpuniverse/evaluator/mcpmark/notion/utils/__init__.py
234
+ mcpuniverse/evaluator/mcpmark/notion/utils/notion_utils.py
235
+ mcpuniverse/evaluator/mcpmark/playwright/eval_web/cloudflare_turnstile_challenge/verify.py
236
+ mcpuniverse/evaluator/mcpmark/playwright/eval_web/extraction_table/verify.py
237
+ mcpuniverse/evaluator/mcpmark/playwright/web_search/birth_of_arvinxu/verify.py
238
+ mcpuniverse/evaluator/mcpmark/playwright/web_search/r1_arxiv/verify.py
239
+ mcpuniverse/evaluator/mcpmark/playwright_webarena/reddit/ai_data_analyst/verify.py
240
+ mcpuniverse/evaluator/mcpmark/playwright_webarena/reddit/budget_europe_travel/verify.py
241
+ mcpuniverse/evaluator/mcpmark/playwright_webarena/reddit/buyitforlife_research/verify.py
242
+ mcpuniverse/evaluator/mcpmark/playwright_webarena/reddit/llm_research_summary/verify.py
243
+ mcpuniverse/evaluator/mcpmark/playwright_webarena/reddit/movie_reviewer_analysis/verify.py
244
+ mcpuniverse/evaluator/mcpmark/playwright_webarena/reddit/nba_statistics_analysis/verify.py
245
+ mcpuniverse/evaluator/mcpmark/playwright_webarena/reddit/routine_tracker_forum/verify.py
246
+ mcpuniverse/evaluator/mcpmark/playwright_webarena/shopping/advanced_product_analysis/verify.py
247
+ mcpuniverse/evaluator/mcpmark/playwright_webarena/shopping/gaming_accessories_analysis/verify.py
248
+ mcpuniverse/evaluator/mcpmark/playwright_webarena/shopping/health_routine_optimization/verify.py
249
+ mcpuniverse/evaluator/mcpmark/playwright_webarena/shopping/holiday_baking_competition/verify.py
250
+ mcpuniverse/evaluator/mcpmark/playwright_webarena/shopping/multi_category_budget_analysis/verify.py
251
+ mcpuniverse/evaluator/mcpmark/playwright_webarena/shopping/printer_keyboard_search/verify.py
252
+ mcpuniverse/evaluator/mcpmark/playwright_webarena/shopping/running_shoes_purchase/verify.py
253
+ mcpuniverse/evaluator/mcpmark/playwright_webarena/shopping_admin/customer_segmentation_setup/verify.py
254
+ mcpuniverse/evaluator/mcpmark/playwright_webarena/shopping_admin/fitness_promotion_strategy/verify.py
255
+ mcpuniverse/evaluator/mcpmark/playwright_webarena/shopping_admin/marketing_customer_analysis/verify.py
256
+ mcpuniverse/evaluator/mcpmark/playwright_webarena/shopping_admin/ny_expansion_analysis/verify.py
257
+ mcpuniverse/evaluator/mcpmark/playwright_webarena/shopping_admin/products_sales_analysis/verify.py
258
+ mcpuniverse/evaluator/mcpmark/playwright_webarena/shopping_admin/sales_inventory_analysis/verify.py
259
+ mcpuniverse/evaluator/mcpmark/playwright_webarena/shopping_admin/search_filtering_operations/verify.py
260
+ mcpuniverse/evaluator/mcpmark/postgres/chinook/customer_data_migration/verify.py
261
+ mcpuniverse/evaluator/mcpmark/postgres/chinook/employee_hierarchy_management/verify.py
262
+ mcpuniverse/evaluator/mcpmark/postgres/chinook/sales_and_music_charts/verify.py
263
+ mcpuniverse/evaluator/mcpmark/postgres/dvdrental/customer_analysis_fix/verify.py
264
+ mcpuniverse/evaluator/mcpmark/postgres/dvdrental/customer_analytics_optimization/verify.py
265
+ mcpuniverse/evaluator/mcpmark/postgres/dvdrental/film_inventory_management/verify.py
266
+ mcpuniverse/evaluator/mcpmark/postgres/employees/employee_demographics_report/verify.py
267
+ mcpuniverse/evaluator/mcpmark/postgres/employees/employee_performance_analysis/verify.py
268
+ mcpuniverse/evaluator/mcpmark/postgres/employees/employee_project_tracking/verify.py
269
+ mcpuniverse/evaluator/mcpmark/postgres/employees/employee_retention_analysis/verify.py
270
+ mcpuniverse/evaluator/mcpmark/postgres/employees/executive_dashboard_automation/verify.py
271
+ mcpuniverse/evaluator/mcpmark/postgres/employees/management_structure_analysis/verify.py
272
+ mcpuniverse/evaluator/mcpmark/postgres/lego/consistency_enforcement/verify.py
273
+ mcpuniverse/evaluator/mcpmark/postgres/lego/database_security_policies/verify.py
274
+ mcpuniverse/evaluator/mcpmark/postgres/lego/transactional_inventory_transfer/verify.py
275
+ mcpuniverse/evaluator/mcpmark/postgres/security/rls_business_access/prepare_environment.py
276
+ mcpuniverse/evaluator/mcpmark/postgres/security/rls_business_access/verify.py
277
+ mcpuniverse/evaluator/mcpmark/postgres/security/user_permission_audit/prepare_environment.py
278
+ mcpuniverse/evaluator/mcpmark/postgres/security/user_permission_audit/verify.py
279
+ mcpuniverse/evaluator/mcpmark/postgres/sports/baseball_player_analysis/verify.py
280
+ mcpuniverse/evaluator/mcpmark/postgres/sports/participant_report_optimization/verify.py
281
+ mcpuniverse/evaluator/mcpmark/postgres/sports/team_roster_management/verify.py
282
+ mcpuniverse/evaluator/mcpmark/postgres/vectors/vectors_setup.py
283
+ mcpuniverse/evaluator/mcpmark/postgres/vectors/dba_vector_analysis/prepare_environment.py
284
+ mcpuniverse/evaluator/mcpmark/postgres/vectors/dba_vector_analysis/verify.py
285
+ mcpuniverse/evaluator/notion/__init__.py
286
+ mcpuniverse/evaluator/notion/functions.py
287
+ mcpuniverse/evaluator/playwright/__init__.py
288
+ mcpuniverse/evaluator/playwright/functions.py
289
+ mcpuniverse/evaluator/weather/__init__.py
290
+ mcpuniverse/evaluator/weather/functions.py
291
+ mcpuniverse/evaluator/yfinance/__init__.py
292
+ mcpuniverse/evaluator/yfinance/functions.py
293
+ mcpuniverse/extensions/mcpplus/__init__.py
294
+ mcpuniverse/extensions/mcpplus/agent/__init__.py
295
+ mcpuniverse/extensions/mcpplus/agent/react_postprocess_agent.py
296
+ mcpuniverse/extensions/mcpplus/benchmark/benchmark_runner.py
297
+ mcpuniverse/extensions/mcpplus/benchmark/builder.py
298
+ mcpuniverse/extensions/mcpplus/benchmark/report.py
299
+ mcpuniverse/extensions/mcpplus/examples/basic_wrapper.py
300
+ mcpuniverse/extensions/mcpplus/examples/benchmark_integration.py
301
+ mcpuniverse/extensions/mcpplus/tools/__init__.py
302
+ mcpuniverse/extensions/mcpplus/tools/proxy_server.py
303
+ mcpuniverse/extensions/mcpplus/tools/wrap_mcp_config.py
304
+ mcpuniverse/extensions/mcpplus/utils/__init__.py
305
+ mcpuniverse/extensions/mcpplus/utils/safe_executor.py
306
+ mcpuniverse/extensions/mcpplus/utils/stats.py
307
+ mcpuniverse/extensions/mcpplus/utils/tracer_analyzer.py
308
+ mcpuniverse/extensions/mcpplus/wrapper/__init__.py
309
+ mcpuniverse/extensions/mcpplus/wrapper/wrapper_manager.py
310
+ mcpuniverse/llm/__init__.py
311
+ mcpuniverse/llm/base.py
312
+ mcpuniverse/llm/claude.py
313
+ mcpuniverse/llm/claude_gateway.py
314
+ mcpuniverse/llm/claude_wr.py
315
+ mcpuniverse/llm/deepseek.py
316
+ mcpuniverse/llm/gemini.py
317
+ mcpuniverse/llm/grok.py
318
+ mcpuniverse/llm/local_llm.py
319
+ mcpuniverse/llm/manager.py
320
+ mcpuniverse/llm/mistral.py
321
+ mcpuniverse/llm/ollama.py
322
+ mcpuniverse/llm/openai.py
323
+ mcpuniverse/llm/openai_agent.py
324
+ mcpuniverse/llm/openrouter.py
325
+ mcpuniverse/llm/sf_llm_express_gateway.py
326
+ mcpuniverse/llm/sf_research_gateway.py
327
+ mcpuniverse/llm/utils.py
328
+ mcpuniverse/llm/tito/__init__.py
329
+ mcpuniverse/llm/tito/engine.py
330
+ mcpuniverse/llm/tito/manager.py
331
+ mcpuniverse/llm/tito/wrapper.py
332
+ mcpuniverse/mcp/__init__.py
333
+ mcpuniverse/mcp/client.py
334
+ mcpuniverse/mcp/config.py
335
+ mcpuniverse/mcp/gateway.py
336
+ mcpuniverse/mcp/manager.py
337
+ mcpuniverse/mcp/permission.py
338
+ mcpuniverse/mcp/configs/server_list.json
339
+ mcpuniverse/mcp/env_pool/__init__.py
340
+ mcpuniverse/mcp/env_pool/base.py
341
+ mcpuniverse/mcp/env_pool/docker.py
342
+ mcpuniverse/mcp/env_pool/manager.py
343
+ mcpuniverse/mcp/servers/__init__.py
344
+ mcpuniverse/mcp/servers/blender/__init__.py
345
+ mcpuniverse/mcp/servers/blender/__main__.py
346
+ mcpuniverse/mcp/servers/blender/server.py
347
+ mcpuniverse/mcp/servers/date/__init__.py
348
+ mcpuniverse/mcp/servers/date/__main__.py
349
+ mcpuniverse/mcp/servers/date/server.py
350
+ mcpuniverse/mcp/servers/echo/__init__.py
351
+ mcpuniverse/mcp/servers/echo/__main__.py
352
+ mcpuniverse/mcp/servers/echo/server.py
353
+ mcpuniverse/mcp/servers/google_search/__init__.py
354
+ mcpuniverse/mcp/servers/google_search/__main__.py
355
+ mcpuniverse/mcp/servers/google_search/server.py
356
+ mcpuniverse/mcp/servers/google_sheets/__init__.py
357
+ mcpuniverse/mcp/servers/google_sheets/__main__.py
358
+ mcpuniverse/mcp/servers/google_sheets/server.py
359
+ mcpuniverse/mcp/servers/jina_scrape_llm_summary/__init__.py
360
+ mcpuniverse/mcp/servers/jina_scrape_llm_summary/__main__.py
361
+ mcpuniverse/mcp/servers/jina_scrape_llm_summary/server.py
362
+ mcpuniverse/mcp/servers/python_code_sandbox/__init__.py
363
+ mcpuniverse/mcp/servers/python_code_sandbox/__main__.py
364
+ mcpuniverse/mcp/servers/python_code_sandbox/server.py
365
+ mcpuniverse/mcp/servers/serper_search/__init__.py
366
+ mcpuniverse/mcp/servers/serper_search/__main__.py
367
+ mcpuniverse/mcp/servers/serper_search/server.py
368
+ mcpuniverse/mcp/servers/serper_search/utils.py
369
+ mcpuniverse/mcp/servers/weather/__init__.py
370
+ mcpuniverse/mcp/servers/weather/__main__.py
371
+ mcpuniverse/mcp/servers/weather/server.py
372
+ mcpuniverse/mcp/servers/wikipedia/__init__.py
373
+ mcpuniverse/mcp/servers/wikipedia/__main__.py
374
+ mcpuniverse/mcp/servers/wikipedia/server.py
375
+ mcpuniverse/mcp/servers/yahoo_finance/__init__.py
376
+ mcpuniverse/mcp/servers/yahoo_finance/__main__.py
377
+ mcpuniverse/mcp/servers/yahoo_finance/server.py
378
+ mcpuniverse/pipeline/__init__.py
379
+ mcpuniverse/pipeline/__main__.py
380
+ mcpuniverse/pipeline/celery_config.py
381
+ mcpuniverse/pipeline/cli.py
382
+ mcpuniverse/pipeline/launcher.py
383
+ mcpuniverse/pipeline/task.py
384
+ mcpuniverse/pipeline/utils.py
385
+ mcpuniverse/pipeline/worker.py
386
+ mcpuniverse/pipeline/mq/__init__.py
387
+ mcpuniverse/pipeline/mq/base.py
388
+ mcpuniverse/pipeline/mq/factory.py
389
+ mcpuniverse/pipeline/mq/kafka_consumer.py
390
+ mcpuniverse/pipeline/mq/kafka_producer.py
391
+ mcpuniverse/pipeline/mq/rabbitmq_consumer.py
392
+ mcpuniverse/pipeline/mq/rabbitmq_producer.py
393
+ mcpuniverse/rl/__init__.py
394
+ mcpuniverse/rl/config.py
395
+ mcpuniverse/rl/dispatcher.py
396
+ mcpuniverse/rl/runner.py
397
+ mcpuniverse/rl/trace_logger.py
398
+ mcpuniverse/rl/trajectory.py
399
+ mcpuniverse/rl/data/__init__.py
400
+ mcpuniverse/rl/formatters/__init__.py
401
+ mcpuniverse/rl/formatters/base.py
402
+ mcpuniverse/rl/formatters/gpt_oss.py
403
+ mcpuniverse/rl/formatters/qwen3.py
404
+ mcpuniverse/rl/integrations/__init__.py
405
+ mcpuniverse/rl/integrations/verl/__init__.py
406
+ mcpuniverse/rl/integrations/verl/mcp_backend.py
407
+ mcpuniverse/rl/integrations/verl/mcp_dataset.py
408
+ mcpuniverse/rl/integrations/verl/mcp_loop_manager.py
409
+ mcpuniverse/rl/integrations/verl/mcp_reward_manager.py
410
+ mcpuniverse/rl/integrations/verl/utils.py
411
+ mcpuniverse/rl/integrations/verl/config/__init__.py
412
+ mcpuniverse/rl/integrations/verl/fully_async/__init__.py
413
+ mcpuniverse/rl/integrations/verl/fully_async/mcp_async_data.py
414
+ mcpuniverse/rl/integrations/verl/fully_async/mcp_async_main.py
415
+ mcpuniverse/rl/integrations/verl/fully_async/mcp_async_rollouter.py
416
+ mcpuniverse/rl/integrations/verl/fully_async/mcp_async_trainer.py
417
+ mcpuniverse/rl/integrations/verl/fully_async/mcp_async_workers.py
418
+ mcpuniverse/rl/integrations/verl/fully_async/mcp_param_sync.py
419
+ mcpuniverse/rl/integrations/verl/hybrid/__init__.py
420
+ mcpuniverse/rl/integrations/verl/hybrid/mcp_main_ppo.py
421
+ mcpuniverse/rl/integrations/verl/hybrid/mcp_trainer.py
422
+ mcpuniverse/tracer/__init__.py
423
+ mcpuniverse/tracer/tracer.py
424
+ mcpuniverse/tracer/types.py
425
+ mcpuniverse/tracer/collectors/__init__.py
426
+ mcpuniverse/tracer/collectors/base.py
427
+ mcpuniverse/tracer/collectors/file.py
428
+ mcpuniverse/tracer/collectors/memory.py
429
+ mcpuniverse/tracer/collectors/sqlite.py
430
+ mcpuniverse/workflows/__init__.py
431
+ mcpuniverse/workflows/base.py
432
+ mcpuniverse/workflows/builder.py
433
+ mcpuniverse/workflows/chain.py
434
+ mcpuniverse/workflows/evaluator_optimizer.py
435
+ mcpuniverse/workflows/orchestrator.py
436
+ mcpuniverse/workflows/parallelization.py
437
+ mcpuniverse/workflows/router.py
mcpuniverse.egg-info/dependency_links.txt ADDED
@@ -0,0 +1 @@
 
 
1
+
mcpuniverse.egg-info/entry_points.txt ADDED
@@ -0,0 +1,2 @@
 
 
 
1
+ [console_scripts]
2
+ mcp-build-plus = mcpuniverse.extensions.mcpplus.tools.wrap_mcp_config:main
mcpuniverse.egg-info/top_level.txt ADDED
@@ -0,0 +1 @@
 
 
1
+ mcpuniverse
mcpuniverse/__init__.py ADDED
File without changes
pyproject.toml ADDED
@@ -0,0 +1,157 @@
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
1
+ [build-system]
2
+ requires = ["setuptools", "wheel"]
3
+ build-backend = "setuptools.build_meta"
4
+
5
+ [project]
6
+ name = "mcpuniverse"
7
+ version = "1.1.3"
8
+ authors = [
9
+ {name = "Salesforce Research"}
10
+ ]
11
+ description = "A framework for developing and benchmarking AI agents using Model Context Protocol (MCP)"
12
+ readme = "README.md"
13
+ license = {text = "Apache 2.0"}
14
+ keywords = [
15
+ "AI",
16
+ "Agents",
17
+ "MCP",
18
+ "benchmarking",
19
+ "LLM",
20
+ "machine-learning"
21
+ ]
22
+ classifiers = [
23
+ "Intended Audience :: Developers",
24
+ "Programming Language :: Python :: 3",
25
+ "Programming Language :: Python :: 3.10",
26
+ "Programming Language :: Python :: 3.11",
27
+ "Programming Language :: Python :: 3.12",
28
+ ]
29
+ requires-python = ">=3.10,<4"
30
+ dependencies = [
31
+ "requests==2.32.4",
32
+ "pydantic==2.11.7",
33
+ "pydantic[email]==2.11.7",
34
+ "mcp==1.13.1",
35
+ "httpx==0.28.1",
36
+ "click==8.1.8",
37
+ "jinja2==3.1.6",
38
+ "python-dotenv==1.0.1",
39
+ "anyio==4.9.0",
40
+ "openai==1.106.1",
41
+ "anthropic==0.49.0",
42
+ "mistralai==1.6.0",
43
+ "pyyaml==6.0.2",
44
+ "google-genai==1.16.1",
45
+ "redis==6.1.0",
46
+ "fastapi==0.115.12",
47
+ "uvicorn[standard]==0.34.0",
48
+ "bcrypt==4.3.0",
49
+ "pyseto==1.8.4",
50
+ "celery==5.5.3",
51
+ "xai-sdk==1.0.0",
52
+ "claude-code-sdk==0.0.20",
53
+ "openai-agents==0.2.11",
54
+ "wikipedia-api==0.8.1",
55
+ "mcp_server_fetch",
56
+ "google-auth==2.38.0",
57
+ "google-auth-oauthlib==1.2.1",
58
+ "google-api-python-client",
59
+ "mcp_server_calculator==0.1.1",
60
+ "yfinance==0.2.61",
61
+ "blender-mcp==1.1.3",
62
+ "playwright==1.52.0",
63
+ "mathutils==3.3.0",
64
+ "pytz==2024.2",
65
+ "tiktoken==0.11.0",
66
+ "kafka-python==2.2.15",
67
+ "pika==1.3.2",
68
+ "tenacity==9.1.2",
69
+ "loguru==0.7.3",
70
+ "aiohttp>=3.9.0",
71
+ "omegaconf>=2.3.0",
72
+ "beautifulsoup4>=4.12.0",
73
+ "pandas>=2.0.0",
74
+ "numpy>=1.24.0",
75
+ "notion-client==2.7.0",
76
+ "sqlalchemy[asyncio]==2.0.41"
77
+ ]
78
+
79
+ [project.optional-dependencies]
80
+ dev = [
81
+ "pytest",
82
+ "pytest-asyncio",
83
+ "pytest-postgresql",
84
+ "pylint",
85
+ "pre-commit",
86
+ ]
87
+ web = [
88
+ "uvicorn[standard]==0.34.0",
89
+ "fastapi==0.115.12",
90
+ "celery==5.5.3",
91
+ "redis==6.1.0",
92
+ "psycopg[binary]==3.2.9",
93
+ "sqlalchemy[asyncio]==2.0.41",
94
+ ]
95
+ dashboard = [
96
+ "gradio>=5.42.0",
97
+ ]
98
+ deep-research = [
99
+ "pillow==12.1.0",
100
+ "datasets==4.5.0",
101
+ "openpyxl==3.1.5",
102
+ "vertexai==1.71.1",
103
+ ]
104
+ vllm = [
105
+ "vllm>=0.15.0",
106
+ "ray>=2.0.0",
107
+ "flash-attn>=2.7.0",
108
+ "torch>=2.6.0",
109
+ "nest_asyncio>=1.5.0",
110
+ ]
111
+ sglang = [
112
+ "sglang>=0.4.0",
113
+ "ray>=2.0.0",
114
+ "flash-attn>=2.7.0",
115
+ "torch>=2.6.0",
116
+ ]
117
+ rl = [
118
+ "verl @ git+https://github.com/verl-project/verl.git@9433f8a8f2771256ea4f8f94e4401bcfe9703228",
119
+ "torch>=2.6.0",
120
+ "ray>=2.0.0",
121
+ "tensordict>=0.6.0",
122
+ "hydra-core>=1.3.0",
123
+ "tqdm>=4.60.0",
124
+ "transformers>=4.40.0",
125
+ "numpy>=1.24.0",
126
+ "accelerate>=1.0.0",
127
+ "peft>=0.10.0",
128
+ "wandb>=0.15.0",
129
+ "datasets>=4.0.0",
130
+ ]
131
+ # System dependency: Node.js/npx is required for some MCP servers
132
+ # (google-maps, slack, notion, postgres, etc.)
133
+ # Install via: conda install -c conda-forge nodejs
134
+ # or: apt-get install nodejs npm
135
+
136
+ [project.scripts]
137
+ mcp-build-plus = "mcpuniverse.extensions.mcpplus.tools.wrap_mcp_config:main"
138
+
139
+ [project.urls]
140
+ Homepage = "https://github.com/SalesforceAIResearch/MCP-Universe"
141
+ Repository = "https://github.com/SalesforceAIResearch/MCP-Universe"
142
+
143
+ [tool.setuptools.package-dir]
144
+ mcpuniverse = "mcpuniverse"
145
+
146
+ [tool.setuptools.packages.find]
147
+ include = ["mcpuniverse*"]
148
+
149
+ [tool.setuptools.package-data]
150
+ mcpuniverse = [
151
+ "agent/configs/*",
152
+ "app/db/migration/*",
153
+ "benchmark/configs/*",
154
+ "dashboard/static/*",
155
+ "dashboard/templates/*",
156
+ "mcp/configs/*"
157
+ ]
pytest.ini ADDED
@@ -0,0 +1,4 @@
 
 
 
 
 
1
+ [pytest]
2
+ testpaths =
3
+ tests
4
+ addopts = -v
requirements.txt ADDED
@@ -0,0 +1,69 @@
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
1
+ requests==2.32.4
2
+ pydantic==2.11.7
3
+ pydantic[email]==2.11.7
4
+ mcp==1.13.1
5
+ httpx==0.28.1
6
+ click==8.1.8
7
+ jinja2==3.1.6
8
+ python-dotenv==1.0.1
9
+ anyio==4.9.0
10
+ openai==1.106.1
11
+ anthropic==0.49.0
12
+ mistralai==1.6.0
13
+ pyyaml==6.0.2
14
+ google-genai==1.16.1
15
+ redis==6.1.0
16
+ psycopg[binary]==3.2.9
17
+ sqlalchemy[asyncio]==2.0.41
18
+ fastapi==0.115.12
19
+ uvicorn[standard]==0.34.0
20
+ bcrypt==4.3.0
21
+ pyseto==1.8.4
22
+ celery==5.5.3
23
+ pytz==2024.2
24
+ xai-sdk==1.0.0
25
+ claude-code-sdk==0.0.20
26
+ openai-agents==0.2.11
27
+ tiktoken==0.11.0
28
+ kafka-python==2.2.15
29
+ pika==1.3.2
30
+ loguru==0.7.3
31
+ aiohttp>=3.9.0
32
+ omegaconf>=2.3.0
33
+ beautifulsoup4>=4.12.0
34
+ pandas>=2.0.0
35
+ numpy>=1.24.0
36
+ tenacity==9.1.2
37
+
38
+ # MCP servers
39
+ # Node.js/npx is required for some MCP servers (google-maps, slack, notion, etc.)
40
+ # Install via: conda install -c conda-forge nodejs
41
+ # or: apt-get install nodejs npm
42
+ wikipedia-api==0.8.1
43
+ mcp_server_fetch
44
+ google-auth==2.38.0
45
+ google-auth-oauthlib==1.2.1
46
+ google-api-python-client
47
+ mcp_server_calculator==0.1.1
48
+ yfinance==0.2.61
49
+ blender-mcp==1.1.3
50
+ playwright==1.52.0
51
+ mathutils==3.3.0
52
+
53
+ # MCPMark
54
+ notion-client==2.7.0
55
+
56
+ # Deep research
57
+ pillow==12.1.0
58
+ datasets==4.5.0
59
+ openpyxl==3.1.5
60
+ vertexai==1.71.1
61
+
62
+ # Local inference backends (optional):
63
+ # pip install mcpuniverse[vllm] — vLLM backend
64
+ # pip install mcpuniverse[sglang] — SGLang backend
65
+ # vllm>=0.15.0
66
+ # sglang>=0.4.0
67
+ # ray>=2.0.0
68
+ # flash-attn>=2.7.0
69
+ # torch>=2.6.0
run_smoke_blender.py ADDED
@@ -0,0 +1,52 @@
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
1
+ """Smoke test: run a single 3d_design task via the existing Blender MCP socket."""
2
+ import asyncio
3
+ import os
4
+ from pathlib import Path
5
+
6
+ import yaml
7
+
8
+ from mcpuniverse.tracer.collectors import FileCollector
9
+ from mcpuniverse.benchmark.runner import BenchmarkRunner
10
+ from mcpuniverse.benchmark.report import BenchmarkReport
11
+ from mcpuniverse.callbacks.handlers.vprint import get_vprint_callbacks
12
+
13
+ SMOKE_YAML = "mcpuniverse/3d_design_smoke.yaml"
14
+
15
+
16
+ def build_smoke_yaml(num_tasks: int = 1):
17
+ src = Path("mcpuniverse/benchmark/configs/mcpuniverse/3d_design.yaml")
18
+ dst = Path("mcpuniverse/benchmark/configs") / SMOKE_YAML
19
+ docs = list(yaml.safe_load_all(src.read_text()))
20
+ for d in docs:
21
+ if d.get("kind") == "benchmark":
22
+ d["spec"]["tasks"] = d["spec"]["tasks"][:num_tasks]
23
+ with dst.open("w") as f:
24
+ yaml.safe_dump_all(docs, f, sort_keys=False)
25
+ print(f"[smoke] wrote {dst} ({num_tasks} task(s))")
26
+
27
+
28
+ async def main():
29
+ assert os.environ.get("OPENAI_API_KEY"), "OPENAI_API_KEY required"
30
+ assert os.environ.get("BLENDER_APP_PATH"), "BLENDER_APP_PATH required"
31
+ assert os.environ.get("MCPUniverse_DIR"), "MCPUniverse_DIR required"
32
+
33
+ build_smoke_yaml()
34
+ Path("log/mcpuniverse").mkdir(parents=True, exist_ok=True)
35
+ tc = FileCollector(log_file="log/mcpuniverse/3d_design_smoke.log")
36
+ runner = BenchmarkRunner(SMOKE_YAML)
37
+ results = await runner.run(trace_collector=tc, callbacks=get_vprint_callbacks())
38
+ BenchmarkReport(runner, trace_collector=tc).dump()
39
+
40
+ print("=" * 66)
41
+ print("Smoke run result")
42
+ print("=" * 66)
43
+ for task_name, task_result in results[0].task_results.items():
44
+ print(f"\nTask: {task_name}")
45
+ for e in task_result["evaluation_results"]:
46
+ ok = "PASS" if e.passed else "FAIL"
47
+ reason = f" reason: {e.reason}" if not e.passed and e.reason else ""
48
+ print(f" [{ok}] op={e.config.op}{reason}")
49
+
50
+
51
+ if __name__ == "__main__":
52
+ asyncio.run(main())
run_smoke_browser.py ADDED
@@ -0,0 +1,48 @@
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
1
+ """Smoke test: run a single browser_automation task."""
2
+ import asyncio
3
+ import os
4
+ from pathlib import Path
5
+
6
+ import yaml
7
+
8
+ from mcpuniverse.tracer.collectors import FileCollector
9
+ from mcpuniverse.benchmark.runner import BenchmarkRunner
10
+ from mcpuniverse.benchmark.report import BenchmarkReport
11
+ from mcpuniverse.callbacks.handlers.vprint import get_vprint_callbacks
12
+
13
+ SMOKE_YAML = "mcpuniverse/browser_automation_smoke.yaml"
14
+
15
+
16
+ def build_smoke_yaml(num_tasks: int = 1):
17
+ src = Path("mcpuniverse/benchmark/configs/mcpuniverse/browser_automation.yaml")
18
+ dst = Path("mcpuniverse/benchmark/configs") / SMOKE_YAML
19
+ docs = list(yaml.safe_load_all(src.read_text()))
20
+ for d in docs:
21
+ if d.get("kind") == "benchmark":
22
+ d["spec"]["tasks"] = d["spec"]["tasks"][:num_tasks]
23
+ with dst.open("w") as f:
24
+ yaml.safe_dump_all(docs, f, sort_keys=False)
25
+ print(f"[smoke] wrote {dst} ({num_tasks} task(s))")
26
+
27
+
28
+ async def main():
29
+ assert os.environ.get("OPENAI_API_KEY"), "OPENAI_API_KEY required"
30
+ build_smoke_yaml()
31
+ Path("log/mcpuniverse").mkdir(parents=True, exist_ok=True)
32
+ tc = FileCollector(log_file="log/mcpuniverse/browser_automation_smoke.log")
33
+ runner = BenchmarkRunner(SMOKE_YAML)
34
+ results = await runner.run(trace_collector=tc, callbacks=get_vprint_callbacks())
35
+ BenchmarkReport(runner, trace_collector=tc).dump()
36
+ print("=" * 66)
37
+ print("Smoke run result")
38
+ print("=" * 66)
39
+ for task_name, task_result in results[0].task_results.items():
40
+ print(f"\nTask: {task_name}")
41
+ for e in task_result["evaluation_results"]:
42
+ ok = "PASS" if e.passed else "FAIL"
43
+ reason = f" reason: {(e.reason or '')[:200]}" if not e.passed and e.reason else ""
44
+ print(f" [{ok}] op={e.config.op}{reason}")
45
+
46
+
47
+ if __name__ == "__main__":
48
+ asyncio.run(main())
run_smoke_financial.py ADDED
@@ -0,0 +1,45 @@
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
1
+ """Smoke test: run a single financial_analysis task to verify the pipeline."""
2
+ import asyncio
3
+ import yaml
4
+ from pathlib import Path
5
+
6
+ from mcpuniverse.tracer.collectors import FileCollector
7
+ from mcpuniverse.benchmark.runner import BenchmarkRunner
8
+ from mcpuniverse.benchmark.report import BenchmarkReport
9
+ from mcpuniverse.callbacks.handlers.vprint import get_vprint_callbacks
10
+
11
+ SMOKE_YAML = "mcpuniverse/financial_analysis_smoke.yaml"
12
+
13
+
14
+ def build_smoke_yaml(num_tasks: int = 1):
15
+ src = Path("mcpuniverse/benchmark/configs/mcpuniverse/financial_analysis.yaml")
16
+ dst = Path("mcpuniverse/benchmark/configs") / SMOKE_YAML
17
+ docs = list(yaml.safe_load_all(src.read_text()))
18
+ for d in docs:
19
+ if d.get("kind") == "benchmark":
20
+ d["spec"]["tasks"] = d["spec"]["tasks"][:num_tasks]
21
+ with dst.open("w") as f:
22
+ yaml.safe_dump_all(docs, f, sort_keys=False)
23
+ print(f"[smoke] wrote {dst} ({num_tasks} task(s))")
24
+
25
+
26
+ async def main():
27
+ build_smoke_yaml()
28
+ Path("log/mcpuniverse").mkdir(parents=True, exist_ok=True)
29
+ tc = FileCollector(log_file="log/mcpuniverse/financial_analysis_smoke.log")
30
+ runner = BenchmarkRunner(SMOKE_YAML)
31
+ results = await runner.run(trace_collector=tc, callbacks=get_vprint_callbacks())
32
+ BenchmarkReport(runner, trace_collector=tc).dump()
33
+ print("=" * 66)
34
+ print("Smoke run result")
35
+ print("=" * 66)
36
+ for task_name, task_result in results[0].task_results.items():
37
+ print(f"\nTask: {task_name}")
38
+ evals = task_result["evaluation_results"]
39
+ for e in evals:
40
+ ok = "PASS" if e.passed else "FAIL"
41
+ print(f" [{ok}] func={e.config.func} op={e.config.op}")
42
+
43
+
44
+ if __name__ == "__main__":
45
+ asyncio.run(main())
setup_blender_and_vnc.sh ADDED
@@ -0,0 +1,422 @@
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
1
+ #!/bin/bash
2
+
3
+ # MCP-Universe Services Launch Script
4
+ # NOTE: It is a REFERENCE script and we can not guarantee it will work in all environments!
5
+
6
+ # This script starts the Blender instance with the MCP addon.
7
+ # It also starts a VNC server, and noVNC web interface for debugging.
8
+ # It will automatically download and setup Blender and noVNC if not present
9
+
10
+ # Color codes for output
11
+ RED='\033[0;31m'
12
+ GREEN='\033[0;32m'
13
+ YELLOW='\033[1;33m'
14
+ BLUE='\033[0;34m'
15
+ NC='\033[0m' # No Color
16
+
17
+ # !!! Change your configuration here !!!
18
+ # Configuration
19
+ DISPLAY_NUMBER=":99"
20
+ DISPLAY_NUM="99"
21
+ VNC_PORT="5999"
22
+ NOVNC_PORT="6080"
23
+ BLENDER_MCP_PORT="9876"
24
+ VNC_DIR="novnc"
25
+ BLENDER_VERSION="4.4.0"
26
+ BLENDER_DOWNLOAD_URL="https://download.blender.org/release/Blender4.4/blender-4.4.0-linux-x64.tar.xz"
27
+ BLENDER_INSTALL_DIR="applications"
28
+ BLENDER_PATH="$BLENDER_INSTALL_DIR/blender-$BLENDER_VERSION-linux-x64/blender"
29
+ BLENDER_ADDON="blender_addon.py"
30
+ LOG_DIR="mcp_services_logs"
31
+ NOVNC_REPO="https://github.com/novnc/noVNC.git"
32
+ # !!! End of configuration !!!
33
+
34
+ # Create necessary directories
35
+ mkdir -p "$LOG_DIR"
36
+ mkdir -p "$BLENDER_INSTALL_DIR"
37
+ mkdir -p "$VNC_DIR"
38
+
39
+ echo -e "${BLUE}========================================${NC}"
40
+ echo -e "${BLUE}MCP-Universe Services Launch Script${NC}"
41
+ echo -e "${BLUE}========================================${NC}"
42
+ echo ""
43
+
44
+ # Function to check if a process is running
45
+ check_process() {
46
+ if pgrep -f "$1" > /dev/null; then
47
+ return 0
48
+ else
49
+ return 1
50
+ fi
51
+ }
52
+
53
+ # Function to wait for a port to be available
54
+ wait_for_port() {
55
+ local port=$1
56
+ local timeout=30
57
+ local count=0
58
+
59
+ echo -e "${YELLOW}Waiting for port $port to be available...${NC}"
60
+ while [ $count -lt $timeout ]; do
61
+ if ss -tuln 2>/dev/null | grep -q ":$port " || netstat -tuln 2>/dev/null | grep -q ":$port "; then
62
+ echo -e "${GREEN}✓ Port $port is available${NC}"
63
+ return 0
64
+ fi
65
+ sleep 1
66
+ count=$((count + 1))
67
+ done
68
+
69
+ echo -e "${RED}✗ Timeout waiting for port $port${NC}"
70
+ return 1
71
+ }
72
+
73
+ # 0. Setup: Download and install dependencies if needed
74
+ echo -e "${YELLOW}[0/4] Checking and installing dependencies...${NC}"
75
+
76
+ # Check and install Blender
77
+ if [ ! -f "$BLENDER_PATH" ]; then
78
+ echo -e "${YELLOW}Blender not found. Downloading and installing...${NC}"
79
+
80
+ cd "$BLENDER_INSTALL_DIR" || exit 1
81
+
82
+ # Download Blender
83
+ echo -e "${BLUE}Downloading Blender $BLENDER_VERSION...${NC}"
84
+ wget -O blender.tar.xz "$BLENDER_DOWNLOAD_URL" 2>&1 | tee "$LOG_DIR/blender_download.log"
85
+
86
+ if [ $? -ne 0 ]; then
87
+ echo -e "${RED}✗ Failed to download Blender${NC}"
88
+ exit 1
89
+ fi
90
+
91
+ # Extract Blender
92
+ echo -e "${BLUE}Extracting Blender...${NC}"
93
+ tar -xf blender.tar.xz
94
+
95
+ if [ $? -ne 0 ]; then
96
+ echo -e "${RED}✗ Failed to extract Blender${NC}"
97
+ exit 1
98
+ fi
99
+
100
+ # Clean up
101
+ rm blender.tar.xz
102
+
103
+ # Verify installation
104
+ if [ -f "$BLENDER_PATH" ]; then
105
+ echo -e "${GREEN}✓ Blender installed successfully at $BLENDER_PATH${NC}"
106
+ else
107
+ echo -e "${RED}✗ Blender installation failed${NC}"
108
+ exit 1
109
+ fi
110
+ else
111
+ echo -e "${GREEN}✓ Blender already installed at $BLENDER_PATH${NC}"
112
+ fi
113
+
114
+ # Check and install noVNC
115
+ if [ ! -d "$VNC_DIR/noVNC" ]; then
116
+ echo -e "${YELLOW}noVNC not found. Cloning repository...${NC}"
117
+
118
+ cd "$VNC_DIR" || exit 1
119
+
120
+ # Clone noVNC
121
+ echo -e "${BLUE}Cloning noVNC...${NC}"
122
+ git clone "$NOVNC_REPO" 2>&1 | tee "$LOG_DIR/novnc_clone.log"
123
+
124
+ if [ $? -ne 0 ]; then
125
+ echo -e "${RED}✗ Failed to clone noVNC${NC}"
126
+ exit 1
127
+ fi
128
+
129
+ # Verify installation
130
+ if [ -f "$VNC_DIR/noVNC/utils/novnc_proxy" ]; then
131
+ echo -e "${GREEN}✓ noVNC installed successfully${NC}"
132
+ else
133
+ echo -e "${RED}✗ noVNC installation failed${NC}"
134
+ exit 1
135
+ fi
136
+ else
137
+ echo -e "${GREEN}✓ noVNC already installed at $VNC_DIR/noVNC${NC}"
138
+ fi
139
+
140
+ # Verify addon exists
141
+ if [ ! -f "$BLENDER_ADDON" ]; then
142
+ echo -e "${RED}✗ Blender addon not found at $BLENDER_ADDON${NC}"
143
+ echo -e "${YELLOW}Please ensure MCP-Universe is cloned to /root/MCP-Universe${NC}"
144
+ exit 1
145
+ else
146
+ echo -e "${GREEN}✓ Blender MCP addon found${NC}"
147
+ fi
148
+
149
+ echo ""
150
+
151
+ # 1. Start VNC Server (provides X display on configured display number)
152
+ echo -e "${YELLOW}[1/4] Checking/Starting VNC Server on display $DISPLAY_NUMBER...${NC}"
153
+
154
+ # Check if Xvfb is running (conflict with VNC)
155
+ if check_process "Xvfb $DISPLAY_NUMBER"; then
156
+ echo -e "${YELLOW}Found Xvfb on $DISPLAY_NUMBER, stopping it (VNC required)...${NC}"
157
+ pkill -f "Xvfb $DISPLAY_NUMBER" || true
158
+ sleep 3
159
+ fi
160
+
161
+ # Check if VNC server is already running
162
+ if check_process "Xvnc $DISPLAY_NUMBER"; then
163
+ echo -e "${GREEN}✓ VNC Server is already running on $DISPLAY_NUMBER${NC}"
164
+ else
165
+ # Kill any existing VNC server
166
+ vncserver -kill $DISPLAY_NUMBER 2>/dev/null || true
167
+ sleep 3
168
+
169
+ # Start VNC server (provides both X server and VNC access)
170
+ OUTPUT=$(vncserver $DISPLAY_NUMBER -localhost -geometry 1920x1080 -depth 24 2>&1)
171
+ VNC_EXIT_CODE=$?
172
+ echo "$OUTPUT" > "$LOG_DIR/vncserver.log"
173
+
174
+ # Check if the output indicates success
175
+ if echo "$OUTPUT" | grep -q "New.*server.*on port"; then
176
+ echo -e "${GREEN}✓ VNC Server started successfully on $DISPLAY_NUMBER${NC}"
177
+ elif [ $VNC_EXIT_CODE -ne 0 ]; then
178
+ echo -e "${RED}✗ Failed to start VNC Server${NC}"
179
+ echo -e "${YELLOW}Log output:${NC}"
180
+ echo "$OUTPUT"
181
+ exit 1
182
+ else
183
+ # Wait and verify process is running
184
+ sleep 5
185
+
186
+ if check_process "Xvnc" || check_process "Xtigervnc"; then
187
+ echo -e "${GREEN}✓ VNC Server started successfully on $DISPLAY_NUMBER${NC}"
188
+ else
189
+ echo -e "${RED}✗ VNC Server process not running${NC}"
190
+ echo "$OUTPUT"
191
+ exit 1
192
+ fi
193
+ fi
194
+ fi
195
+
196
+ export DISPLAY=$DISPLAY_NUMBER
197
+ echo -e "${BLUE}DISPLAY set to $DISPLAY_NUMBER (VNC port: $VNC_PORT)${NC}"
198
+ echo ""
199
+
200
+ # 2. Start noVNC
201
+ echo -e "${YELLOW}[2/4] Starting noVNC web interface...${NC}"
202
+
203
+ # Kill existing noVNC processes to avoid duplicates
204
+ pkill -f novnc_proxy 2>/dev/null || true
205
+ sleep 1
206
+
207
+ if check_process "novnc_proxy"; then
208
+ echo -e "${YELLOW}noVNC still running, force killing...${NC}"
209
+ pkill -9 -f novnc_proxy 2>/dev/null || true
210
+ sleep 1
211
+ fi
212
+
213
+ cd "$VNC_DIR" || { echo -e "${RED}✗ Cannot access VNC directory${NC}"; exit 1; }
214
+
215
+ # Check if noVNC exists
216
+ if [ ! -f "./noVNC/utils/novnc_proxy" ]; then
217
+ echo -e "${RED}✗ noVNC not found at $VNC_DIR/noVNC${NC}"
218
+ exit 1
219
+ fi
220
+
221
+ # Start noVNC proxy with configured VNC port
222
+ echo -e "${BLUE}Starting noVNC with VNC port: $VNC_PORT${NC}"
223
+ ./noVNC/utils/novnc_proxy --vnc localhost:$VNC_PORT --listen localhost:$NOVNC_PORT > "$LOG_DIR/novnc.log" 2>&1 &
224
+ sleep 3
225
+
226
+ if check_process "novnc_proxy"; then
227
+ echo -e "${GREEN}✓ noVNC started successfully${NC}"
228
+ wait_for_port $NOVNC_PORT || echo -e "${YELLOW} Warning: Port $NOVNC_PORT check timed out but process is running${NC}"
229
+ else
230
+ echo -e "${RED}✗ Failed to start noVNC${NC}"
231
+ cat "$LOG_DIR/novnc.log"
232
+ exit 1
233
+ fi
234
+
235
+ cd - > /dev/null
236
+
237
+ echo -e "${BLUE}noVNC web interface: http://localhost:$NOVNC_PORT/vnc.html${NC}"
238
+ echo ""
239
+
240
+ # 3. Start Blender with MCP addon
241
+ echo -e "${YELLOW}[3/4] Starting Blender with MCP addon...${NC}"
242
+
243
+ # Kill any existing Blender processes
244
+ if check_process "blender"; then
245
+ echo -e "${YELLOW}Stopping existing Blender instances...${NC}"
246
+ pkill -f blender || true
247
+ sleep 2
248
+ fi
249
+
250
+ # Verify Blender exists
251
+ if [ ! -f "$BLENDER_PATH" ]; then
252
+ echo -e "${RED}✗ Blender not found at $BLENDER_PATH${NC}"
253
+ exit 1
254
+ fi
255
+
256
+ # Verify addon exists
257
+ if [ ! -f "$BLENDER_ADDON" ]; then
258
+ echo -e "${RED}✗ Blender addon not found at $BLENDER_ADDON${NC}"
259
+ exit 1
260
+ fi
261
+
262
+ # Create a startup script that loads the addon and keeps Blender running
263
+ cat > /tmp/blender_startup.py << 'STARTUP_EOF'
264
+ import bpy
265
+ import sys
266
+ import time
267
+
268
+ # Get the addon path from command line arguments
269
+ addon_path = sys.argv[-1] if sys.argv[-1].endswith('.py') else None
270
+
271
+ if addon_path:
272
+ print(f"Loading Blender MCP addon from: {addon_path}")
273
+
274
+ # Load and execute the addon
275
+ with open(addon_path, 'r') as f:
276
+ addon_code = f.read()
277
+
278
+ # Execute the addon code in the global namespace
279
+ exec(addon_code, globals())
280
+
281
+ print("Blender MCP addon loaded successfully")
282
+ else:
283
+ print("ERROR: No addon path provided")
284
+ sys.exit(1)
285
+
286
+ # Keep Blender running with a timer
287
+ def keep_alive():
288
+ return 1.0 # Return to be called again in 1 second
289
+
290
+ bpy.app.timers.register(keep_alive, persistent=True)
291
+
292
+ print("Blender is now running with MCP addon. Press Ctrl+C in terminal to stop.")
293
+ STARTUP_EOF
294
+
295
+ # Start Blender with the startup script
296
+ echo -e "${BLUE}Starting Blender with MCP addon...${NC}"
297
+ DISPLAY=$DISPLAY_NUMBER "$BLENDER_PATH" --python /tmp/blender_startup.py -- "$BLENDER_ADDON" > "$LOG_DIR/blender.log" 2>&1 &
298
+ BLENDER_PID=$!
299
+ echo -e "${BLUE}Started Blender with PID: $BLENDER_PID${NC}"
300
+
301
+ # Wait for Blender to start
302
+ sleep 8
303
+
304
+ if check_process "blender"; then
305
+ echo -e "${GREEN}✓ Blender is running${NC}"
306
+ else
307
+ echo -e "${RED}✗ Blender process not found${NC}"
308
+ echo -e "${YELLOW}Last 30 lines of Blender log:${NC}"
309
+ tail -30 "$LOG_DIR/blender.log"
310
+ exit 1
311
+ fi
312
+ echo ""
313
+
314
+ # 4. Verify Blender MCP addon is loaded
315
+ echo -e "${YELLOW}[4/4] Verifying Blender MCP addon...${NC}"
316
+ sleep 5
317
+
318
+ # Create a Python verification script
319
+ cat > /tmp/verify_blender_addon.py << 'VERIFY_EOF'
320
+ #!/usr/bin/env python3
321
+ import socket
322
+ import json
323
+ import time
324
+ import sys
325
+
326
+ def verify_blender_mcp():
327
+ """Verify that the Blender MCP addon is loaded and responding"""
328
+
329
+ import os
330
+ port = int(os.environ.get('BLENDER_MCP_PORT', '9876'))
331
+ host = 'localhost'
332
+
333
+ print(f"Checking Blender MCP server on {host}:{port}...")
334
+
335
+ max_attempts = 15
336
+ for attempt in range(max_attempts):
337
+ try:
338
+ sock = socket.socket(socket.AF_INET, socket.SOCK_STREAM)
339
+ sock.settimeout(2)
340
+ result = sock.connect_ex((host, port))
341
+
342
+ if result == 0:
343
+ print(f"✓ SUCCESS: Blender MCP server is running on port {port}")
344
+ print(f"✓ The addon is loaded correctly")
345
+
346
+ # Try to send a test command
347
+ try:
348
+ test_command = {
349
+ "jsonrpc": "2.0",
350
+ "method": "list_tools",
351
+ "params": {},
352
+ "id": 1
353
+ }
354
+ sock.sendall((json.dumps(test_command) + '\n').encode())
355
+ sock.settimeout(5)
356
+ response = sock.recv(4096).decode()
357
+
358
+ if response:
359
+ print(f"✓ Server responded to test command. (It's fine if the server responded an error - we are only testing connectivity here)")
360
+ if len(response) > 200:
361
+ print(f" Response preview: {response[:200]}...")
362
+ else:
363
+ print(f" Response: {response}")
364
+
365
+ except Exception as e:
366
+ print(f" Note: Connection successful but command test failed: {e}")
367
+
368
+ sock.close()
369
+ return True
370
+ else:
371
+ sock.close()
372
+ print(f" Attempt {attempt + 1}/{max_attempts}: Port not open yet, waiting...")
373
+ time.sleep(3)
374
+
375
+ except Exception as e:
376
+ print(f" Attempt {attempt + 1}/{max_attempts}: {e}")
377
+ time.sleep(3)
378
+
379
+ print(f"✗ FAILED: Could not connect to Blender MCP server on port {port}")
380
+ return False
381
+
382
+ if __name__ == "__main__":
383
+ success = verify_blender_mcp()
384
+ sys.exit(0 if success else 1)
385
+ VERIFY_EOF
386
+
387
+ chmod +x /tmp/verify_blender_addon.py
388
+
389
+ # Run the verification script with port as environment variable
390
+ if BLENDER_MCP_PORT=$BLENDER_MCP_PORT python3 /tmp/verify_blender_addon.py; then
391
+ echo -e "${GREEN}✓ Blender MCP addon verification successful${NC}"
392
+ else
393
+ echo -e "${RED}✗ Blender MCP addon verification failed${NC}"
394
+ echo -e "${YELLOW}Blender log (last 30 lines):${NC}"
395
+ tail -30 "$LOG_DIR/blender.log"
396
+ fi
397
+ echo ""
398
+
399
+ # Summary
400
+ echo -e "${BLUE}========================================${NC}"
401
+ echo -e "${BLUE}Service Launch Summary${NC}"
402
+ echo -e "${BLUE}========================================${NC}"
403
+ echo -e "${GREEN}Services Status:${NC}"
404
+ echo ""
405
+ echo -e " ${GREEN}✓${NC} VNC Server (Display $DISPLAY_NUMBER) - Running"
406
+ echo -e " ${GREEN}✓${NC} noVNC Web Interface - Running"
407
+ echo -e " ${GREEN}✓${NC} Blender with MCP addon - Running"
408
+ echo ""
409
+ echo -e "${BLUE}Access Information:${NC}"
410
+ echo -e " • noVNC URL: ${BLUE}http://localhost:$NOVNC_PORT/vnc.html${NC}"
411
+ echo -e " • Blender MCP: ${BLUE}localhost:$BLENDER_MCP_PORT${NC}"
412
+ echo -e " • Display: ${BLUE}$DISPLAY_NUMBER${NC}"
413
+ echo ""
414
+ echo -e "${BLUE}Log Files:${NC}"
415
+ echo -e " • VNC: ${BLUE}$LOG_DIR/vncserver.log${NC}"
416
+ echo -e " • noVNC: ${BLUE}$LOG_DIR/novnc.log${NC}"
417
+ echo -e " • Blender: ${BLUE}$LOG_DIR/blender.log${NC}"
418
+ echo ""
419
+ echo -e "${YELLOW}Commands:${NC}"
420
+ echo -e " Stop all: ${YELLOW}vncserver -kill $DISPLAY_NUMBER && pkill novnc_proxy && pkill blender${NC}"
421
+ echo -e " View logs: ${YELLOW}tail -f $LOG_DIR/blender.log${NC}"
422
+ echo -e "${BLUE}========================================${NC}"
slime_mcp_rollout/.env.example ADDED
@@ -0,0 +1,100 @@
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
1
+ # slime_mcp_rollout — environment variables placeholder
2
+ #
3
+ # Copy this file to `slime_mcp_rollout/.env` and fill in the keys you
4
+ # need. `.env` is git-ignored, the example is committed.
5
+ #
6
+ # Loading: mcpuniverse calls `load_dotenv()` automatically on import, so
7
+ # either of these will pick it up:
8
+ # - source slime_mcp_rollout/.env # explicit shell-level export
9
+ # - placing the file at the repo root as `.env`
10
+ # - (programmatic) `dotenv.load_dotenv("slime_mcp_rollout/.env")` before
11
+ # calling `synthesis.synthesize(...)` or `rollout.generate(...)`
12
+ #
13
+ # Detailed acquisition steps + free-tier limits + verification curl
14
+ # commands per key: see docs/API_KEYS.md
15
+ # Per-domain prerequisites (Blender / Chrome / Docker / Xvfb): see
16
+ # docs/PREREQUISITES.md
17
+
18
+ # =============================================================================
19
+ # Agent LLM (required for ALL domains)
20
+ # =============================================================================
21
+ # The LLM driving the ReAct/FunctionCall loop. Set EITHER:
22
+ # - DEEPSEEK_API_KEY (default; base_url defaults to https://api.deepseek.com/v1)
23
+ # - OPENAI_API_KEY (when base_url is set to OpenAI or a local SGLang)
24
+ #
25
+ # Both can coexist; build_llm() picks DEEPSEEK_API_KEY first, then OPENAI_API_KEY.
26
+ DEEPSEEK_API_KEY=
27
+ OPENAI_API_KEY=
28
+
29
+ # =============================================================================
30
+ # web_search (google-search + fetch MCP servers)
31
+ # =============================================================================
32
+ # SerpAPI — free tier: 250 searches/month, sign up at https://serpapi.com
33
+ SERP_API_KEY=
34
+ #
35
+ # IMPORTANT: web_search's evaluator uses gpt-4.1 as an LLM-as-judge
36
+ # (hardcoded in mcpuniverse/evaluator/google_search/functions.py:47).
37
+ # Even if your agent runs on DeepSeek, you MUST also set OPENAI_API_KEY
38
+ # above, otherwise every web_search task fails with a generic
39
+ # "Execution error". See docs/API_KEYS.md for details.
40
+
41
+ # =============================================================================
42
+ # location_navigation (google-maps MCP server)
43
+ # =============================================================================
44
+ # Google Cloud → APIs & Services → Credentials → API key.
45
+ # Enable: Geocoding, Places (New), Directions, Distance Matrix.
46
+ # $200/month free credit covers ~100 full sweeps. Key starts with "AIzaSy...".
47
+ GOOGLE_MAPS_API_KEY=
48
+
49
+ # =============================================================================
50
+ # repository_management (github MCP server, runs in Docker)
51
+ # =============================================================================
52
+ # **Use a dedicated throwaway GitHub account.** Tasks create / modify /
53
+ # delete real repositories under this account.
54
+ #
55
+ # Scopes needed on the classic PAT: repo, delete_repo, workflow, read:org, gist.
56
+ # Token starts with "ghp_...".
57
+ GITHUB_PERSONAL_ACCESS_TOKEN=
58
+ #
59
+ # The GitHub username whose namespace tasks operate in (not the email).
60
+ # Task JSON templates expand `{{GITHUB_PERSONAL_ACCOUNT_NAME}}` to this.
61
+ GITHUB_PERSONAL_ACCOUNT_NAME=
62
+ #
63
+ # Infrastructure: Docker daemon must be running before launching repo
64
+ # rollouts (the github MCP server runs as `docker run
65
+ # ghcr.io/github/github-mcp-server:0.5.0`). In sandboxed environments
66
+ # with a TLS-inspecting egress proxy, the handler auto-mounts
67
+ # /etc/ssl/certs/ca-certificates.crt into the container.
68
+
69
+ # =============================================================================
70
+ # 3d_design (blender MCP server)
71
+ # =============================================================================
72
+ # No API key. Two paths the evaluator needs to know about:
73
+ #
74
+ # Absolute path to the Blender 4.4 executable (NOT a wrapper script).
75
+ # The evaluator launches headless instances to validate .blend files.
76
+ BLENDER_APP_PATH=
77
+ #
78
+ # Absolute path to this repo root (used by the evaluator to locate
79
+ # reference .blend files under mcpuniverse/evaluator/blender/).
80
+ MCPUniverse_DIR=
81
+
82
+ # =============================================================================
83
+ # financial_analysis (yfinance + calculator MCP servers)
84
+ # =============================================================================
85
+ # Zero env vars required. yfinance scrapes Yahoo Finance HTML directly.
86
+ # See docs/FINANCE_PITFALLS.md for throttling traps.
87
+
88
+ # =============================================================================
89
+ # browser_automation (playwright + date MCP servers)
90
+ # =============================================================================
91
+ # Zero env vars required. Needs Node 18+ and Google Chrome on the host
92
+ # (see docs/PREREQUISITES.md for the `npx playwright install chrome` step).
93
+
94
+ # =============================================================================
95
+ # multi_server (NOT YET IMPLEMENTED)
96
+ # =============================================================================
97
+ # Combines two of the above servers per sub-task. Some combinations require
98
+ # Notion (NOTION_TOKEN + workspace setup). See
99
+ # mcpuniverse/benchmark/configs/mcpmark/README.md for Notion-side setup.
100
+ # NOTION_TOKEN=
slime_mcp_rollout/README.md ADDED
@@ -0,0 +1,364 @@
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
1
+ # slime_mcp_rollout
2
+
3
+ Thin adapter that runs MCP-Universe benchmarks via a single async entry,
4
+ usable two ways:
5
+
6
+ 1. **`rollout.generate(args, sample, sampling_params)`** — the
7
+ [slime](https://github.com/THUDM/slime) `custom_generate_function`
8
+ contract. One sample in, one `{response, reward, meta, ...}` out.
9
+ 2. **`synthesis.synthesize(domain, ...)`** — library + CLI for offline
10
+ trajectory generation; same engine, batched with `asyncio.gather`.
11
+
12
+ Both share the same handler stack (`tasks/*.py`). **Zero modifications to
13
+ `mcpuniverse/`** — all customization lives in this folder.
14
+
15
+ ## 📦 Sample rollout data on HuggingFace
16
+
17
+ [`Shuibai12138/mcp-universe-trajectories`](https://huggingface.co/datasets/Shuibai12138/mcp-universe-trajectories)
18
+ — `deepseek-v4-pro` × `financial_analysis` × 40 tasks (60% pass).
19
+
20
+ Three views, pick by need:
21
+
22
+ | Want | Where | UX |
23
+ |------|-------|----|
24
+ | **Browse / SQL-filter slim summaries** | [HF viewer table ↗](https://huggingface.co/datasets/Shuibai12138/mcp-universe-trajectories/viewer) | one row per task |
25
+ | **Iteration-grouped view** (recommended for eyeballing ReAct flow) | [`markdown_grouped/_index.md` ↗](https://huggingface.co/datasets/Shuibai12138/mcp-universe-trajectories/blob/main/markdown_grouped/_index.md) → click any task | each `### Iteration N` bundles 1 LLM call + its tool calls; tool responses collapsed in `<details>` |
26
+ | **Flat step view** | [`markdown/_index.md` ↗](https://huggingface.co/datasets/Shuibai12138/mcp-universe-trajectories/blob/main/markdown/_index.md) → click any task | one section per tracer span |
27
+ | **Programmatic access to raw trace** | [`trajectories/*.json` ↗](https://huggingface.co/datasets/Shuibai12138/mcp-universe-trajectories/tree/main/trajectories) | full nested JSON |
28
+
29
+ Generate Markdown views from a local run:
30
+ ```bash
31
+ python3 -m slime_mcp_rollout.data.render_trajectories # flat (markdown/)
32
+ python3 -m slime_mcp_rollout.data.render_trajectories --group-by-iteration # grouped (markdown_grouped/)
33
+ python3 -m slime_mcp_rollout.data.render_trajectories --both
34
+ ```
35
+
36
+ ---
37
+
38
+ ## Layout
39
+
40
+ ```
41
+ slime_mcp_rollout/
42
+ ├── README.md ← this file
43
+ ├── rollout.py ← ★ slime entry: async def generate(args, sample, sampling_params)
44
+ ├── synthesis.py ← ★ offline runner: async def synthesize(...) + CLI
45
+ ├── llm_bridge.py ← FastRetryOpenAIModel — shorter retry envelope around upstream
46
+ │
47
+ ├── tasks/
48
+ │ ├── base.py ← BaseRealHandler + AgentPool + RolloutResult; the actual rollout loop
49
+ │ ├── __init__.py ← strict domain registry (no auto-Placeholder)
50
+ │ ├── financial_analysis.py ← yfinance + calculator MCP servers
51
+ │ ├── d3_design.py ← blender MCP server (multi-port pool for concurrency)
52
+ │ ├── blender_pool.py ← per-process Blender pool + BlenderConnection port monkey-patch
53
+ │ ├── browser_automation.py ← playwright + date MCP servers
54
+ │ ├── web_search.py ← google-search + fetch MCP servers
55
+ │ ├── location_navigation.py ← google-maps MCP server
56
+ │ ├── repository_management.py ← github (docker) MCP server + CA-bundle monkey-patch
57
+ │ └── yfinance_cache.py ← per-process memoization + retry for yfinance throttling
58
+ │
59
+ ├── data/
60
+ │ ├── prepare_data.py ← official-task JSON → slime-style JSONL converter
61
+ │ ├── render_trajectories.py ← trajectory → Markdown views
62
+ │ ├── <domain>_sample.jsonl ← generated, one per domain
63
+ │ └── trajectories/<run-id>/ ← synthesis output (per-task json + _summary.json + trajectories.jsonl)
64
+ │
65
+ ├── docs/
66
+ │ ├── PREREQUISITES.md ← per-domain system setup matrix
67
+ │ ├── API_KEYS.md ← how to obtain SerpAPI / Google-Maps / GitHub-PAT / OpenAI keys
68
+ │ ├── EVALUATORS.md ← per-domain ground-truth source + 4 evaluator patterns
69
+ │ ├── CONCURRENCY.md ← what's shared between samples, per-domain ceilings
70
+ │ └── FINANCE_PITFALLS.md ← yfinance throttling, -1 mystery, double-querying, …
71
+ │
72
+ └── tests/
73
+ └── test_local_rollout.py ← legacy single-sample smoke driver
74
+ ```
75
+
76
+ ## Domain status
77
+
78
+ Registry keys, `sample.metadata.category`, and `task_id` prefixes all use
79
+ **MCP-Universe's official domain names** (= yaml file basenames under
80
+ `mcpuniverse/benchmark/configs/mcpuniverse/`).
81
+
82
+ | Domain | Status | Primary MCP servers | Required env / setup |
83
+ |----------------------------|----------|---------------------------|----------------------|
84
+ | `financial_analysis` | ✅ | yfinance, calculator | none |
85
+ | `3d_design` | ✅ | blender | Blender 4.4 + Xvfb + addon (see PREREQUISITES) |
86
+ | `browser_automation` | ✅ | playwright, date | Node 18+ + Chrome |
87
+ | `web_search` | ✅ | google-search, fetch | `SERP_API_KEY` + `OPENAI_API_KEY` (LLM-as-judge) |
88
+ | `location_navigation` | ✅ | google-maps | `GOOGLE_MAPS_API_KEY` |
89
+ | `repository_management` | ✅ | github (Docker) | `GITHUB_PERSONAL_ACCESS_TOKEN`, `GITHUB_PERSONAL_ACCOUNT_NAME`, running docker daemon |
90
+ | `multi_server` | ⛔ raises | (mixed: + Notion) | not yet implemented |
91
+
92
+ Setup details → [`docs/PREREQUISITES.md`](docs/PREREQUISITES.md).
93
+ API-key acquisition → [`docs/API_KEYS.md`](docs/API_KEYS.md).
94
+ Concurrency safety per domain → [`docs/CONCURRENCY.md`](docs/CONCURRENCY.md).
95
+
96
+ ---
97
+
98
+ ## What gets reused from MCP-Universe
99
+
100
+ | Piece | Source (read-only — never edited) |
101
+ |-----------------------------|----------------------------------------------------------------------|
102
+ | **Agent loop** | `mcpuniverse.agent.react.ReAct` / `mcpuniverse.agent.function_call.FunctionCall` |
103
+ | **MCP server lifecycle** | `mcpuniverse.mcp.manager.MCPManager` (stdio spawn, tool registry, transport retry) |
104
+ | **Task spec + evaluators** | `mcpuniverse.benchmark.task.Task` + `mcpuniverse/evaluator/<domain>/functions.py` |
105
+ | **LLM client** | `mcpuniverse.llm.openai.OpenAIModel` (subclassed to `FastRetryOpenAIModel`) |
106
+ | **Reward rule (all-pass)** | mirrored from `mcpuniverse/rl/trajectory.py:710-723` (all `eval_passed` must be True for reward=1) |
107
+ | **Eval data (ground-truth)**| `mcpuniverse/benchmark/configs/mcpuniverse/<domain>/*.json` |
108
+ | **Tracer + MemoryCollector**| `mcpuniverse.tracer.Tracer` / `mcpuniverse.callbacks.memory.MemoryCollector` |
109
+
110
+ What lives in this folder instead of upstream:
111
+
112
+ | Concern | Where in `slime_mcp_rollout/` |
113
+ |---------------------------------|------------------------------|
114
+ | Per-agent pool (concurrent rollouts) | `tasks/base.py` `AgentPool` |
115
+ | 3d_design multi-Blender pool | `tasks/blender_pool.py` |
116
+ | GitHub MCP CA-bundle mount | `tasks/repository_management.py` (monkey-patches `MCPManager.__init__`) |
117
+ | yfinance call-level memoization + retry | `tasks/yfinance_cache.py` |
118
+ | Shorter LLM retry envelope | `llm_bridge.FastRetryOpenAIModel` (overrides `generate_async` defaults) |
119
+ | Cleanup contract | `BaseRealHandler.rollout` always builds a Tracer+MemoryCollector and passes the real records into `task.reset(records)` |
120
+
121
+ ---
122
+
123
+ ## Where the agent trajectory actually executes
124
+
125
+ `BaseRealHandler.rollout()` in `tasks/base.py` is the engine — same for
126
+ slime and for synthesis. Sketch:
127
+
128
+ ```
129
+ rollout.generate(...) # slime entry
130
+ └─ get_handler(sample).rollout(sample, sampling_params, args)
131
+ │ = synthesis.synthesize(...)'s inner call too
132
+ ▼
133
+ BaseRealHandler.rollout (tasks/base.py:rollout)
134
+ │
135
+ ├─ acquire agent from AgentPool ← built once, reused across samples
136
+ │
137
+ │ each agent = ReAct (mcpuniverse.agent.react.ReAct)
138
+ │ OR FunctionCall (mcpuniverse.agent.function_call.FunctionCall)
139
+ │ holding an MCPManager (mcpuniverse.mcp.manager) for the
140
+ │ servers declared in the handler's _default_agent_config().
141
+ │
142
+ ├─ Task(json_path, context) ← mcpuniverse.benchmark.task.Task — loads the
143
+ │ official JSON; owns evaluators + cleanups.
144
+ │
145
+ ├─ Tracer + MemoryCollector ← mcpuniverse.tracer / callbacks.memory
146
+ │
147
+ ├─ agent.change_servers(...) ← if task declares specific servers
148
+ ├─ agent.reset() ← clear prior conversation
149
+ │
150
+ ├─ response = await agent.execute(question, output_format, tracer)
151
+ │ ← THIS is the multi-iter ReAct/FC loop;
152
+ │ lives entirely in mcpuniverse.agent.*
153
+ │
154
+ ├─ eval_results = await task.evaluate(response_str)
155
+ │ ← evaluator code from
156
+ │ mcpuniverse.evaluator.<domain>.functions
157
+ │ (retried up to 3x on evaluator-internal errors)
158
+ │
159
+ ├─ reward = 1.0 if all(eval_passed) else 0.0
160
+ │
161
+ ├─ task.reset(records) ← cleanup; uses real trace records
162
+ │ (e.g. github → delete_repository keyed on
163
+ │ each create_repository call seen in the trace)
164
+ │
165
+ └─ release agent back to pool
166
+ ```
167
+
168
+ The *only* parts implemented here (not reused) are the orchestration
169
+ (pool, retry, cleanup wiring, trace plumbing) — every "real" inference,
170
+ every tool call, every evaluator score is upstream code.
171
+
172
+ ---
173
+
174
+ ## API #1 — `synthesis` (offline trajectory generation)
175
+
176
+ ### Library
177
+
178
+ ```python
179
+ import asyncio
180
+ from slime_mcp_rollout.synthesis import synthesize
181
+
182
+ summary = asyncio.run(synthesize(
183
+ domain="financial_analysis",
184
+ num=10, # or: task_ids=["financial_analysis_0001", ...]
185
+ model_name="deepseek-v4-pro",
186
+ base_url="https://api.deepseek.com/v1",
187
+ api_key=None, # default: DEEPSEEK_API_KEY / OPENAI_API_KEY env
188
+ collect_trace=True, # write per-task .json + jsonl
189
+ concurrency=4, # asyncio.gather fan-out
190
+ out_dir=None, # default: data/trajectories/<model>__<domain>__<ts>/
191
+ verbose=True,
192
+ ))
193
+ print(summary["pass_rate_adjusted"])
194
+ ```
195
+
196
+ ### CLI
197
+
198
+ ```bash
199
+ export DEEPSEEK_API_KEY=sk-...
200
+ export PYTHONPATH=.
201
+
202
+ python3 -m slime_mcp_rollout.synthesis \
203
+ --domain financial_analysis \
204
+ -n 40 \
205
+ --concurrency 8 \
206
+ --collect-trace
207
+ ```
208
+
209
+ Output layout:
210
+ ```
211
+ data/trajectories/deepseek-v4-pro__financial_analysis__2026-05-24_141237/
212
+ ├── _summary.json ← pass/fail counts, elapsed, cost, failed_tasks[]
213
+ ├── trajectories.jsonl ← one line per task: response, reward, meta, trajectory
214
+ ├── financial_analysis_0001.json ← per-task full trace
215
+ ├── financial_analysis_0002.json
216
+ └── ...
217
+ ```
218
+
219
+ The CLI's `--task-ids T1 T2 ...` overrides `-n`. See
220
+ [`docs/CONCURRENCY.md`](docs/CONCURRENCY.md) for safe ceilings per domain.
221
+
222
+ ---
223
+
224
+ ## API #2 — `rollout.generate` (slime hook)
225
+
226
+ Wire it into slime via:
227
+
228
+ ```bash
229
+ slime ... \
230
+ --custom_generate_function_path slime_mcp_rollout.rollout.generate \
231
+ --rollout_extra_args '{"sglang_url": "http://localhost:30000/v1",
232
+ "model_name": "Qwen3-...",
233
+ "api_key": "EMPTY"}' \
234
+ --data_path slime_mcp_rollout/data/financial_analysis_sample.jsonl
235
+ ```
236
+
237
+ Per-sample contract:
238
+
239
+ ```python
240
+ async def generate(args, sample, sampling_params=None):
241
+ # sample (dict) MUST have:
242
+ # "task_id": str
243
+ # "prompt": str (the human question — usually identical to the task json's "question")
244
+ # "metadata": {"category": <domain>, "json_path": <path to mcpuniverse task json>, ...}
245
+ #
246
+ # args carries slime's optional overrides:
247
+ # base_url / sglang_url → endpoint for the trained model
248
+ # model_name → name SGLang advertises
249
+ # api_key → any string for SGLang
250
+ #
251
+ # After return, sample is mutated in-place with:
252
+ # sample["response"], sample["reward"] (float in [0,1]), sample["meta"]
253
+ return sample
254
+ ```
255
+
256
+ The first call lazily builds the LLM client + handler registry behind an
257
+ `asyncio.Lock`; subsequent calls reuse them.
258
+
259
+ ---
260
+
261
+ ## Data — where do samples come from, how to add/modify
262
+
263
+ ### Default flow
264
+
265
+ `data/prepare_data.py` converts every JSON under
266
+ `mcpuniverse/benchmark/configs/mcpuniverse/<domain>/` into one line of
267
+ `data/<domain>_sample.jsonl`:
268
+
269
+ ```bash
270
+ python3 -m slime_mcp_rollout.data.prepare_data --domain web_search
271
+ # wrote 50 samples → slime_mcp_rollout/data/web_search_sample.jsonl
272
+ ```
273
+
274
+ Each line:
275
+ ```json
276
+ {
277
+ "task_id": "web_search_0001",
278
+ "prompt": "<the question text>",
279
+ "metadata": {
280
+ "category": "web_search",
281
+ "json_path": "/abs/.../mcpuniverse/benchmark/configs/mcpuniverse/web_search/info_search_task_0001.json",
282
+ "output_format": {...},
283
+ "mcp_servers": [...]
284
+ }
285
+ }
286
+ ```
287
+
288
+ Only `task_id`, `prompt`, and `metadata.category` + `metadata.json_path`
289
+ are load-bearing. `prompt` is what the agent sees; `json_path` is what
290
+ `Task(...)` loads to learn the evaluators + cleanups.
291
+
292
+ ### To add custom samples
293
+
294
+ You can either:
295
+ - **Drop new JSONs upstream** under
296
+ `mcpuniverse/benchmark/configs/mcpuniverse/<domain>/`, then re-run
297
+ `prepare_data.py`. (No need to touch any code — the converter is
298
+ generic.)
299
+ - **Hand-write a JSONL** directly into `data/<domain>_sample.jsonl`. The
300
+ `json_path` you point at decides which evaluator runs; the rest of the
301
+ sample is informational.
302
+
303
+ ### To change the question without changing the evaluator
304
+
305
+ `prompt` is what the LLM sees. If you edit `prompt` in the JSONL but
306
+ keep `metadata.json_path` pointing at the original task JSON, the
307
+ evaluator still grades against the upstream ground-truth — useful for
308
+ robustness testing the model's wording sensitivity.
309
+
310
+ ---
311
+
312
+ ## Ground-truth — where it lives, what's reproducible
313
+
314
+ 📖 **Full per-domain breakdown**: [`docs/EVALUATORS.md`](docs/EVALUATORS.md)
315
+ covers the four evaluator patterns (hardcoded value / re-query API /
316
+ re-execute / LLM-as-judge), reproducibility caveats, and per-domain
317
+ links into the upstream code.
318
+
319
+ Short version of where to edit if you need to override:
320
+
321
+ | Edit you want | File to touch |
322
+ |----------------------------------------------|----------------------------------------------------------------------------|
323
+ | The expected answer for one task | `mcpuniverse/benchmark/configs/mcpuniverse/<domain>/<task>.json` — `evaluators[*].op_args.value` |
324
+ | The check function itself (e.g. exact → fuzzy match) | `mcpuniverse/evaluator/<domain>/functions.py` — but **only if** you decide modifying upstream is acceptable for your fork |
325
+ | Which LLM judges open-ended `web_search` answers | `mcpuniverse/evaluator/google_search/functions.py:45` — hardcoded `gpt-4.1`; also needs `OPENAI_API_KEY` (see [`docs/API_KEYS.md`](docs/API_KEYS.md)) |
326
+ | Reward aggregation (all-pass → partial credit) | `slime_mcp_rollout/tasks/base.py` `_compute_reward` — local override, no upstream change |
327
+
328
+ The hard rule in this folder: **we never edit
329
+ `mcpuniverse/evaluator/`** — anything domain-specific that the
330
+ benchmark publishes is the source of truth we grade against.
331
+
332
+ ---
333
+
334
+ ## Prerequisites — quick
335
+
336
+ See [`docs/PREREQUISITES.md`](docs/PREREQUISITES.md) for the full
337
+ system-setup matrix (Blender + Xvfb, Chrome, Docker daemon, etc.) and
338
+ [`docs/API_KEYS.md`](docs/API_KEYS.md) for key acquisition. The bare
339
+ common set:
340
+
341
+ ```bash
342
+ pip install -r requirements.txt
343
+ export DEEPSEEK_API_KEY=sk-... # or OPENAI_API_KEY=sk-...
344
+ export PYTHONPATH=.
345
+ ```
346
+
347
+ Per-domain extras (see `docs/PREREQUISITES.md` for the full procedure):
348
+
349
+ - `financial_analysis`: nothing extra (read [`docs/FINANCE_PITFALLS.md`](docs/FINANCE_PITFALLS.md) first — yfinance throttling).
350
+ - `3d_design`: Blender 4.4 + Xvfb + addon loaded on `localhost:9876`.
351
+ - `browser_automation`: Node 18+, Google Chrome.
352
+ - `web_search`: `SERP_API_KEY` **and** `OPENAI_API_KEY` (judge).
353
+ - `location_navigation`: `GOOGLE_MAPS_API_KEY`.
354
+ - `repository_management`: `GITHUB_PERSONAL_ACCESS_TOKEN`, `GITHUB_PERSONAL_ACCOUNT_NAME`, running docker daemon. In sandboxed environments with a TLS-inspecting proxy, the handler auto-mounts `/etc/ssl/certs/ca-certificates.crt` into the github MCP container.
355
+
356
+ ---
357
+
358
+ ## What is NOT yet in this adapter
359
+
360
+ - `multi_server` domain (Notion setup non-trivial; see `mcpuniverse/benchmark/configs/mcpmark/README.md` if you want to extend).
361
+ - Token-level `loss_mask` (slime re-tokenizes; all tokens currently count equally).
362
+ - TITO (token-in token-out) mode.
363
+ - env_pool / Docker container reuse across rollouts (each rollout still spawns its own MCP-server subprocesses).
364
+ - GitHub PAT pool (single PAT today; for >2-4 concurrent rollouts in `repository_management`, MCPMark recommends rotating PATs — not yet wired).
slime_mcp_rollout/__init__.py ADDED
@@ -0,0 +1,5 @@
 
 
 
 
 
 
1
+ """slime_mcp_rollout: thin glue to drive MCP-Universe agents from slime rollout.
2
+
3
+ Stage 1 goal: prove the rollout loop works end-to-end against any OpenAI-compatible
4
+ endpoint (real OpenAI for testing, or slime's local SGLang router in production).
5
+ """
slime_mcp_rollout/llm_bridge.py ADDED
@@ -0,0 +1,177 @@
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
1
+ """LLM bridge.
2
+
3
+ slime trains an SGLang inference server that exposes an OpenAI-compatible
4
+ endpoint. MCP-Universe's OpenAIModel already accepts `base_url`, so this
5
+ module is mainly a factory that points it at the right URL.
6
+
7
+ For stage 1 testing we just hit api.openai.com or api.deepseek.com directly
8
+ with the user's key.
9
+
10
+ Retry-tuning notes
11
+ ------------------
12
+ ``mcpuniverse/llm/base.py:240-242`` (generate_async) reads three knobs from
13
+ ``**kwargs`` via ``.pop()`` and uses them as a retry/timeout envelope around
14
+ the actual OpenAI call::
15
+
16
+ retries = kwargs.pop("retries", 3) # extra attempts
17
+ retry_delay = kwargs.pop("retry_delay", 5) # seconds between attempts
18
+ timeout = kwargs.pop("timeout", 60) # per-attempt wall clock
19
+
20
+ With defaults that is up to ``(3 + 1) * 60 + 3 * 5 = 255 s`` per LLM call
21
+ before propagating an error. A ReAct agent issuing 30-50 such calls in a
22
+ task easily blows the per-task wall-clock budget when even one call hits
23
+ a transient slowdown.
24
+
25
+ The companion sync path (``_generate`` in ``mcpuniverse/llm/openai.py``)
26
+ uses ``kwargs.get`` for the same knobs, which would leak them into the
27
+ OpenAI SDK and trigger ``TypeError: unexpected kwarg``. We don't touch
28
+ the sync path — every agent we use (react/function_call/react_train) is
29
+ on the async path.
30
+
31
+ ``FastRetryOpenAIModel`` only injects defaults; if the agent ever passes
32
+ ``retries=`` / ``retry_delay=`` / ``timeout=`` explicitly, the agent's
33
+ value wins (``setdefault`` semantics).
34
+ """
35
+ import os
36
+ from typing import Any, List, Optional
37
+
38
+ from mcpuniverse.llm.openai import OpenAIModel
39
+ from mcpuniverse.tracer import Tracer
40
+ from mcpuniverse.callbacks.base import BaseCallback
41
+
42
+
43
+ # Tuned defaults: worst case ~ (1 + 1) * 90 + 1 * 2 = 182 s vs. upstream 255 s.
44
+ #
45
+ # History:
46
+ # - timeout=45 was too aggressive: DeepSeek long-context responses
47
+ # legitimately take 60-90 s, and we observed several rollouts where
48
+ # every attempt timed out at 45 s back-to-back, burning ~139 s per
49
+ # LLM call before the agent gave up.
50
+ # - 90 s covers the long tail of DeepSeek inference; one retry catches
51
+ # genuine transient errors (network blip, brief rate-limit) without
52
+ # paying for a third attempt.
53
+ #
54
+ # Worst case 182 s per call × 50 ReAct iterations = ~150 min, still inside
55
+ # our 3600 s per-task budget. Bump timeout further if you see repeated
56
+ # "Timeout on attempt 1/2" warnings during real training.
57
+ _FAST_RETRIES = 1
58
+ _FAST_RETRY_DELAY = 2
59
+ _FAST_TIMEOUT = 90
60
+
61
+
62
+ class FastRetryOpenAIModel(OpenAIModel):
63
+ """OpenAIModel with shorter generate_async retry envelope.
64
+
65
+ Upstream defaults (``retries=3, retry_delay=5, timeout=60``) come from
66
+ ``mcpuniverse/llm/base.py:240``. We override the async path only —
67
+ sync ``_generate`` is unsafe to touch (kwargs leak into OpenAI SDK).
68
+ """
69
+
70
+ async def generate_async(
71
+ self,
72
+ messages: Optional[List[dict]] = None,
73
+ tracer: Optional[Tracer] = None,
74
+ callbacks: Optional[BaseCallback | List[BaseCallback]] = None,
75
+ **kwargs: Any,
76
+ ):
77
+ kwargs.setdefault("retries", _FAST_RETRIES)
78
+ kwargs.setdefault("retry_delay", _FAST_RETRY_DELAY)
79
+ kwargs.setdefault("timeout", _FAST_TIMEOUT)
80
+ return await super().generate_async(
81
+ messages=messages,
82
+ tracer=tracer,
83
+ callbacks=callbacks,
84
+ **kwargs,
85
+ )
86
+
87
+
88
+ def build_llm(
89
+ model_name: str = "deepseek-v4-pro",
90
+ base_url: Optional[str] = None,
91
+ api_key: Optional[str] = None,
92
+ ) -> OpenAIModel:
93
+ """Build a FastRetry OpenAIModel pointed at an OpenAI-compatible endpoint.
94
+
95
+ Args:
96
+ model_name: Model name advertised by the endpoint.
97
+ base_url: OpenAI-compatible endpoint. Defaults to api.openai.com.
98
+ Set to slime's SGLang router (e.g. http://localhost:30000/v1)
99
+ during real training.
100
+ api_key: API key. For SGLang any non-empty string works.
101
+ """
102
+ cfg = {"model_name": model_name}
103
+ if base_url:
104
+ cfg["base_url"] = base_url
105
+ if api_key:
106
+ cfg["api_key"] = api_key
107
+ elif os.environ.get("DEEPSEEK_API_KEY"):
108
+ cfg["api_key"] = os.environ["DEEPSEEK_API_KEY"]
109
+ elif os.environ.get("OPENAI_API_KEY"):
110
+ cfg["api_key"] = os.environ["OPENAI_API_KEY"]
111
+ return FastRetryOpenAIModel(config=cfg)
112
+
113
+
114
+ class MultiKeyLLM:
115
+ """Round-robins ``generate_async`` across N FastRetryOpenAIModel instances,
116
+ each carrying a different API key.
117
+
118
+ DeepSeek rate-limits per key (~30 completions/min). Spreading the rollout's
119
+ LLM calls across K keys multiplies effective throughput ~K× and lets us run
120
+ higher concurrency without any single key throttling (which was causing the
121
+ 600s task timeouts). The agent only calls ``generate_async``; every other
122
+ attribute access is delegated to the first underlying model so the wrapper
123
+ is a drop-in for OpenAIModel.
124
+ """
125
+
126
+ def __init__(self, models: List[OpenAIModel]):
127
+ if not models:
128
+ raise ValueError("MultiKeyLLM needs at least one model")
129
+ self._models = models
130
+ self._idx = 0
131
+
132
+ def _next(self) -> OpenAIModel:
133
+ # asyncio is single-threaded; a plain counter is race-free here.
134
+ m = self._models[self._idx % len(self._models)]
135
+ self._idx += 1
136
+ return m
137
+
138
+ async def generate_async(self, *args: Any, **kwargs: Any):
139
+ return await self._next().generate_async(*args, **kwargs)
140
+
141
+ def set_context(self, context: Any):
142
+ # Fan out context (e.g. env-injected keys) to ALL models, not just
143
+ # models[0] — otherwise the other keys' models would miss it.
144
+ for m in self._models:
145
+ if hasattr(m, "set_context"):
146
+ m.set_context(context)
147
+
148
+ def __getattr__(self, name: str):
149
+ # Delegate misses (model_name, config, sync helpers, ...) to a
150
+ # representative model. Guard against recursion: dunders and the
151
+ # internal `_models` must NOT re-enter __getattr__ (happens during
152
+ # unpickling/copy when `_models` isn't set yet → infinite recursion).
153
+ if name.startswith("__") or name == "_models":
154
+ raise AttributeError(name)
155
+ return getattr(self.__dict__["_models"][0], name)
156
+
157
+
158
+ def build_multi_llm(
159
+ model_name: str = "deepseek-v4-pro",
160
+ base_url: Optional[str] = None,
161
+ api_keys: Optional[List[str]] = None,
162
+ ):
163
+ """Build a MultiKeyLLM over ``api_keys`` (falls back to single-key build_llm
164
+ when only one key is available)."""
165
+ keys = [k.strip() for k in (api_keys or []) if k and k.strip()]
166
+ if not keys:
167
+ raw = os.environ.get("DEEPSEEK_API_KEYS", "")
168
+ keys = [k.strip() for k in raw.split(",") if k.strip()]
169
+ # de-dup, preserve order
170
+ seen = set()
171
+ keys = [k for k in keys if not (k in seen or seen.add(k))]
172
+ if len(keys) <= 1:
173
+ return build_llm(model_name=model_name, base_url=base_url,
174
+ api_key=keys[0] if keys else None)
175
+ models = [build_llm(model_name=model_name, base_url=base_url, api_key=k)
176
+ for k in keys]
177
+ return MultiKeyLLM(models)
slime_mcp_rollout/rollout.py ADDED
@@ -0,0 +1,107 @@
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
1
+ """slime entry point: async def generate(args, sample, sampling_params).
2
+
3
+ Slime calls this for every sample in a rollout batch. We do the minimum
4
+ slime contract:
5
+ 1. Look up the task handler by `sample.metadata.category` or task_id.
6
+ 2. Run the multi-turn MCP loop, get a final response + scalar reward.
7
+ 3. Fill the slime-expected fields on the sample object and return it.
8
+
9
+ Stage 1: `sample` is a plain dict. When integrating with real slime you may
10
+ need to adapt to slime.Sample (just a dataclass with .response / .reward /
11
+ .tokens / .loss_mask attributes).
12
+ """
13
+ from __future__ import annotations
14
+
15
+ import asyncio
16
+ import logging
17
+ import os
18
+ from typing import Any, Dict
19
+
20
+ from .llm_bridge import build_llm
21
+ from .tasks import get_handler, init_default_registry
22
+
23
+ logger = logging.getLogger(__name__)
24
+
25
+ _REGISTRY_INITIALIZED = False
26
+ _LLM = None
27
+ _INIT_LOCK = asyncio.Lock()
28
+
29
+
30
+ async def _maybe_init_registry(args: Dict[str, Any]):
31
+ """Async-safe lazy init. Slime fires many gather()-ed callers; without
32
+ the lock they all see _REGISTRY_INITIALIZED==False and race to build
33
+ duplicate LLMs / registries."""
34
+ global _REGISTRY_INITIALIZED, _LLM
35
+ if _REGISTRY_INITIALIZED:
36
+ return
37
+ async with _INIT_LOCK:
38
+ if _REGISTRY_INITIALIZED:
39
+ return
40
+ args = args or {}
41
+ model_name = args.get("model_name", os.environ.get(
42
+ "SLIME_MCP_MODEL", "deepseek-v4-pro"))
43
+ base_url = (args.get("base_url") or args.get("sglang_url")
44
+ or os.environ.get("SLIME_MCP_BASE_URL"))
45
+ # Prefer a pool of keys (round-robin across them for throughput).
46
+ api_keys = args.get("api_keys")
47
+ if not api_keys:
48
+ raw = os.environ.get("DEEPSEEK_API_KEYS", "")
49
+ api_keys = [k.strip() for k in raw.split(",") if k.strip()]
50
+ if api_keys and len(api_keys) > 1:
51
+ from .llm_bridge import build_multi_llm
52
+ _LLM = build_multi_llm(model_name=model_name, base_url=base_url,
53
+ api_keys=api_keys)
54
+ logger.info("rollout LLM: round-robin over %d DeepSeek keys", len(api_keys))
55
+ else:
56
+ _LLM = build_llm(
57
+ model_name=model_name, base_url=base_url,
58
+ api_key=(args.get("api_key")
59
+ or os.environ.get("DEEPSEEK_API_KEY")
60
+ or os.environ.get("OPENAI_API_KEY")),
61
+ )
62
+ init_default_registry(_LLM)
63
+ _REGISTRY_INITIALIZED = True
64
+
65
+
66
+ async def generate(args, sample, sampling_params=None):
67
+ """slime custom_generate_function entry point.
68
+
69
+ Args:
70
+ args: slime's argparse Namespace or dict-like. We read:
71
+ - sglang_url / base_url
72
+ - model_name
73
+ - api_key (optional)
74
+ sample: dict-like with at least:
75
+ - task_id: str
76
+ - prompt: str
77
+ - metadata: {category, json_path, ...}
78
+ sampling_params: passed through (unused in stage 1; LLM uses MCP-U defaults).
79
+
80
+ Returns the same sample object with:
81
+ sample["response"] = final answer text
82
+ sample["reward"] = float in [0,1]
83
+ sample["meta"] = per-rollout debug info
84
+ """
85
+ # Allow args to be either argparse-Namespace or dict
86
+ if hasattr(args, "__dict__") and not isinstance(args, dict):
87
+ args_dict = vars(args)
88
+ else:
89
+ args_dict = dict(args or {})
90
+
91
+ await _maybe_init_registry(args_dict)
92
+ sampling_params = sampling_params or {}
93
+
94
+ handler = get_handler(sample)
95
+ logger.info("Rollout %s via %s", sample.get("task_id"), handler.name)
96
+
97
+ result = await handler.rollout(sample, sampling_params, args_dict)
98
+
99
+ # Mutate sample with slime-expected fields
100
+ sample["response"] = result.response
101
+ sample["reward"] = result.reward
102
+ sample["meta"] = result.meta
103
+ if result.tokens is not None:
104
+ sample["tokens"] = result.tokens
105
+ if result.loss_mask is not None:
106
+ sample["loss_mask"] = result.loss_mask
107
+ return sample
slime_mcp_rollout/run_rollout_sft.py ADDED
@@ -0,0 +1,78 @@
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
1
+ """Driver: rollout the sft_v1 augmented finance tasks, RESUMABLE + SHARDED.
2
+
3
+ Writes trajectories to a FIXED dir (data/trajectories/sft_final/) and skips any
4
+ task whose trajectory already exists, so it can be relaunched to resume.
5
+
6
+ Multi-process: set NUM_SHARDS=K and SHARD_INDEX=i (0..K-1). Each process handles
7
+ the disjoint slice remaining[i::K]. All shards share the fixed OUT_DIR and the
8
+ on-disk yfinance cache (both cross-process safe). Running K shards as separate
9
+ processes gives K independent asyncio event loops, which is what actually scales
10
+ throughput here (a single process tops out well below what DeepSeek/proxies can
11
+ serve).
12
+
13
+ Env knobs:
14
+ NUM_SHARDS (default 1)
15
+ SHARD_INDEX (default 0)
16
+ ROLLOUT_CONC (per-process concurrency, default 10)
17
+ """
18
+ import asyncio
19
+ import glob
20
+ import os
21
+ from pathlib import Path
22
+
23
+ from dotenv import load_dotenv
24
+
25
+ REPO = Path(__file__).resolve().parents[1]
26
+ load_dotenv(REPO / "slime_mcp_rollout" / ".env")
27
+
28
+ from slime_mcp_rollout.data.prepare_data import prepare # noqa: E402
29
+ from slime_mcp_rollout.synthesis import synthesize # noqa: E402
30
+
31
+ AUG = REPO / "slime_mcp_rollout" / "augment_tasks" / "financial_analysis"
32
+ OUT_DIR = REPO / "slime_mcp_rollout" / "data" / "trajectories" / "sft_final"
33
+
34
+
35
+ def main():
36
+ num_shards = max(1, int(os.environ.get("NUM_SHARDS", "1")))
37
+ shard_index = int(os.environ.get("SHARD_INDEX", "0")) % num_shards
38
+ conc = max(1, int(os.environ.get("ROLLOUT_CONC", "10")))
39
+ tag = f"[shard {shard_index}/{num_shards}]"
40
+
41
+ OUT_DIR.mkdir(parents=True, exist_ok=True)
42
+ all_ids = sorted(f"financial_analysis_{Path(f).stem}"
43
+ for f in glob.glob(str(AUG / "sft_v1*.json")))
44
+ done_ids = {Path(f).stem for f in glob.glob(str(OUT_DIR / "*.json"))
45
+ if not f.endswith("_summary.json")}
46
+ remaining_all = [i for i in all_ids if i not in done_ids]
47
+ # disjoint slice for this shard (sorted+strided → consistent across shards)
48
+ mine = remaining_all[shard_index::num_shards]
49
+ print(f"{tag} total={len(all_ids)} done={len(done_ids)} "
50
+ f"remaining_all={len(remaining_all)} my_shard={len(mine)} conc={conc}",
51
+ flush=True)
52
+ if not mine:
53
+ print(f"{tag} nothing to do", flush=True)
54
+ return
55
+ # Shared sample.jsonl: regenerate only if missing (it already exists from
56
+ # prior runs and the task set is stable). Avoids a cross-shard write race.
57
+ sample_file = REPO / "slime_mcp_rollout" / "data" / "financial_analysis_sample.jsonl"
58
+ if not sample_file.exists():
59
+ prepare("financial_analysis")
60
+ print(f"{tag} prepare ok, rolling out {len(mine)} tasks", flush=True)
61
+ summary = asyncio.run(synthesize(
62
+ domain="financial_analysis",
63
+ task_ids=mine,
64
+ model_name="deepseek-v4-pro",
65
+ api_key=os.environ.get("DEEPSEEK_API_KEY"),
66
+ collect_trace=True,
67
+ concurrency=conc,
68
+ task_timeout_sec=1800,
69
+ out_dir=OUT_DIR,
70
+ verbose=True,
71
+ skip_prewarm=True,
72
+ ))
73
+ print(f"{tag} batch done pass_rate_adjusted={summary.get('pass_rate_adjusted')}",
74
+ flush=True)
75
+
76
+
77
+ if __name__ == "__main__":
78
+ main()
slime_mcp_rollout/synthesis.py ADDED
@@ -0,0 +1,418 @@
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
1
+ """Programmatic + CLI data-synthesis runner.
2
+
3
+ Two entry points:
4
+ - Library: ``await synthesize(domain, num=N, ...)`` — returns summary dict
5
+ - CLI: ``python3 -m slime_mcp_rollout.synthesis --domain ... --num ...``
6
+
7
+ Both delegate to the same single-sample engine (``rollout.generate``) that
8
+ slime's RL rollout uses, so there's a single source of truth for the agent
9
+ loop, evaluator wiring, and trajectory format.
10
+
11
+ Concurrency: pass ``--concurrency N`` (or ``concurrency=N`` kwarg) to run
12
+ N samples in parallel. Each handler owns an agent pool (see
13
+ ``tasks.base.AgentPool``), so concurrent rollouts don't share state.
14
+ Default 1 = sequential (legacy behavior).
15
+ """
16
+ from __future__ import annotations
17
+
18
+ import argparse
19
+ import asyncio
20
+ import datetime as _dt
21
+ import json
22
+ import os
23
+ import re
24
+ import shutil
25
+ import sys
26
+ import time
27
+ import traceback
28
+ from pathlib import Path
29
+ from typing import Any, Dict, Iterable, List, Optional
30
+
31
+ from slime_mcp_rollout.rollout import generate
32
+
33
+ REPO_ROOT = Path(__file__).resolve().parents[1]
34
+ DATA_DIR = REPO_ROOT / "slime_mcp_rollout/data"
35
+ TRAJ_DIR = DATA_DIR / "trajectories"
36
+
37
+
38
+ # ---------------------------------------------------------------------------
39
+ # Helpers
40
+ # ---------------------------------------------------------------------------
41
+ def _safe(name: str) -> str:
42
+ return re.sub(r"[^A-Za-z0-9._-]+", "-", name).strip("-")
43
+
44
+
45
+ def _make_run_dir(model_name: str, domain: str) -> Path:
46
+ ts = _dt.datetime.now().strftime("%Y-%m-%d_%H%M%S")
47
+ run_id = f"{_safe(model_name)}__{domain}__{ts}"
48
+ out = TRAJ_DIR / run_id
49
+ out.mkdir(parents=True, exist_ok=True)
50
+ latest = TRAJ_DIR / "latest"
51
+ try:
52
+ if latest.is_symlink():
53
+ latest.unlink()
54
+ elif latest.is_dir():
55
+ shutil.rmtree(latest)
56
+ elif latest.exists():
57
+ latest.unlink()
58
+ latest.symlink_to(run_id)
59
+ except OSError:
60
+ pass
61
+ return out
62
+
63
+
64
+ def _usage_field(usage_obj, key: str) -> int:
65
+ if usage_obj is None:
66
+ return 0
67
+ if isinstance(usage_obj, dict):
68
+ return int(usage_obj.get(key, 0) or 0)
69
+ return int(getattr(usage_obj, key, 0) or 0)
70
+
71
+
72
+ def _sum_tokens(trajectory: List[Dict[str, Any]]) -> Dict[str, int]:
73
+ in_tok = 0
74
+ out_tok = 0
75
+ for span in trajectory or []:
76
+ for inner in (span or {}).get("records", []) or []:
77
+ data = (inner or {}).get("data", {}) or {}
78
+ if data.get("type") != "llm":
79
+ continue
80
+ response = data.get("response") or {}
81
+ if isinstance(response, dict):
82
+ usage = response.get("usage")
83
+ else:
84
+ usage = getattr(response, "usage", None)
85
+ in_tok += _usage_field(usage, "prompt_tokens")
86
+ out_tok += _usage_field(usage, "completion_tokens")
87
+ return {"in": in_tok, "out": out_tok}
88
+
89
+
90
+ # ---------------------------------------------------------------------------
91
+ # Public API
92
+ # ---------------------------------------------------------------------------
93
+ async def synthesize(
94
+ domain: str,
95
+ *,
96
+ num: int = 1,
97
+ task_ids: Optional[Iterable[str]] = None,
98
+ model_name: str = "deepseek-v4-pro",
99
+ base_url: str = "https://dimcode.cn/v1",
100
+ api_key: Optional[str] = None,
101
+ collect_trace: bool = True,
102
+ concurrency: int = 1,
103
+ out_dir: Optional[Path] = None,
104
+ verbose: bool = True,
105
+ task_timeout_sec: Optional[float] = None,
106
+ skip_prewarm: bool = False,
107
+ prewarm_delay: float = 0.4,
108
+ ) -> Dict[str, Any]:
109
+ """Run rollouts on ``num`` samples (or specific ``task_ids``) of a domain
110
+ and write per-task trajectory JSON + ``_summary.json`` to ``out_dir``.
111
+
112
+ Args:
113
+ domain: official MCP-Universe domain name (e.g. "financial_analysis").
114
+ num: number of samples (ignored if task_ids is given).
115
+ task_ids: explicit list of task_ids to run.
116
+ model_name: passed to llm_bridge.build_llm.
117
+ base_url: OpenAI-compatible endpoint.
118
+ api_key: defaults to DEEPSEEK_API_KEY / OPENAI_API_KEY env.
119
+ collect_trace: write trajectory files; False = fastest, no per-task .json.
120
+ concurrency: run N samples in parallel via asyncio.gather.
121
+ out_dir: trajectory directory; None = auto under data/trajectories/.
122
+ verbose: print per-task progress.
123
+ task_timeout_sec: per-task wall-clock cap. If a single rollout exceeds
124
+ this, it's cancelled and recorded as an error (sibling rollouts under
125
+ gather are unaffected). None = no cap (default). Highly recommended
126
+ for unattended runs because one stuck LLM / MCP call can otherwise
127
+ hold up the whole batch indefinitely.
128
+
129
+ Returns: summary dict (also written to ``<out_dir>/_summary.json``).
130
+ """
131
+ api_key = (api_key or os.environ.get("DEEPSEEK_API_KEY")
132
+ or os.environ.get("OPENAI_API_KEY"))
133
+ if not api_key:
134
+ raise RuntimeError("API key required (DEEPSEEK_API_KEY or OPENAI_API_KEY)")
135
+
136
+ data_file = DATA_DIR / f"{domain}_sample.jsonl"
137
+ if not data_file.exists():
138
+ raise FileNotFoundError(
139
+ f"{data_file} missing. Run:\n"
140
+ f" python3 -m slime_mcp_rollout.data.prepare_data --domain {domain}"
141
+ )
142
+
143
+ samples = [json.loads(l) for l in data_file.read_text().splitlines() if l.strip()]
144
+ if task_ids:
145
+ wanted = set(task_ids)
146
+ samples = [s for s in samples if s.get("task_id") in wanted]
147
+ missing = wanted - {s.get("task_id") for s in samples}
148
+ if missing:
149
+ raise ValueError(f"task_ids not found in {data_file}: {sorted(missing)}")
150
+ else:
151
+ samples = samples[:num]
152
+
153
+ args = {
154
+ "model_name": model_name,
155
+ "base_url": base_url,
156
+ "api_key": api_key,
157
+ "collect_trace": collect_trace,
158
+ "concurrency": concurrency,
159
+ "skip_prewarm": skip_prewarm,
160
+ "prewarm_delay": prewarm_delay,
161
+ }
162
+
163
+ if collect_trace and out_dir is None:
164
+ out_dir = _make_run_dir(model_name, domain)
165
+ if out_dir is not None and verbose:
166
+ print(f"[synth] writing trajectories to {out_dir}", flush=True)
167
+
168
+ # Pre-warm the yfinance cache for finance batches BEFORE the concurrent
169
+ # rollout. Each task's metadata carries json_path → read op_args and
170
+ # serially fetch all referenced tickers/windows (polite, single-threaded,
171
+ # never self-throttles). Concurrent rollout afterwards hits cache only,
172
+ # eliminating Yahoo per-IP throttle "Execution error"s. No-op for
173
+ # domains whose op_args have no ticker (everything but financial_analysis).
174
+ if not bool(args.get("skip_prewarm")):
175
+ try:
176
+ from slime_mcp_rollout.tasks.yfinance_cache import prewarm_from_task_jsons
177
+ json_paths = [
178
+ (s.get("metadata") or {}).get("json_path")
179
+ for s in samples
180
+ if (s.get("metadata") or {}).get("json_path")
181
+ ]
182
+ if json_paths:
183
+ summary_pw = prewarm_from_task_jsons(
184
+ json_paths, delay=float(args.get("prewarm_delay", 0.4)),
185
+ verbose=verbose,
186
+ )
187
+ if verbose and (summary_pw.get("tickers_warmed")
188
+ or summary_pw.get("history_warmed")):
189
+ print(f"[synth] yfinance prewarm: {summary_pw}", flush=True)
190
+ except Exception as e: # pylint: disable=broad-exception-caught
191
+ if verbose:
192
+ print(f"[synth] prewarm skipped ({e})", flush=True)
193
+
194
+ started_at = time.time()
195
+ per_task: List[Dict[str, Any]] = []
196
+
197
+ async def _run_one(idx: int, s: Dict[str, Any]) -> Dict[str, Any]:
198
+ t0 = time.time()
199
+ if verbose:
200
+ print(f"[{idx}/{len(samples)}] start {s['task_id']}", flush=True)
201
+ try:
202
+ if task_timeout_sec is not None:
203
+ out = await asyncio.wait_for(
204
+ generate(args, s, sampling_params={}),
205
+ timeout=task_timeout_sec,
206
+ )
207
+ else:
208
+ out = await generate(args, s, sampling_params={})
209
+ reward = out.get("reward", 0.0)
210
+ meta = out.get("meta") or {}
211
+ trajectory = meta.pop("trajectory", []) or []
212
+ tok = _sum_tokens(trajectory)
213
+ elapsed = round(time.time() - t0, 2)
214
+ if verbose:
215
+ print(
216
+ f"[{idx}/{len(samples)}] done {s['task_id']} "
217
+ f"reward={reward} elapsed={elapsed}s "
218
+ f"trace={len(trajectory)} in={tok['in']} out={tok['out']}",
219
+ flush=True,
220
+ )
221
+ row = {
222
+ "task_id": s["task_id"],
223
+ "domain": domain,
224
+ "model": model_name,
225
+ "prompt": s.get("prompt", ""),
226
+ "response": out.get("response"),
227
+ "reward": reward,
228
+ "elapsed_sec": elapsed,
229
+ "in_tokens": tok["in"],
230
+ "out_tokens": tok["out"],
231
+ "meta": meta,
232
+ "trajectory": trajectory,
233
+ }
234
+ if out_dir is not None:
235
+ # Atomic write: a half-written file (e.g. process killed mid-write
236
+ # during a resume restart) would be counted "done" by the shard
237
+ # resume logic and silently dropped by the SFT filter. Write to a
238
+ # temp file then os.replace (atomic rename) so readers only ever
239
+ # see a complete trajectory.
240
+ pt = out_dir / f"{s['task_id']}.json"
241
+ tmp = out_dir / f".{s['task_id']}.json.tmp"
242
+ tmp.write_text(json.dumps(row, ensure_ascii=False, indent=2, default=str))
243
+ os.replace(tmp, pt)
244
+ return row
245
+ except asyncio.TimeoutError:
246
+ elapsed = round(time.time() - t0, 2)
247
+ if verbose:
248
+ print(
249
+ f"[{idx}/{len(samples)}] TIMEOUT {s['task_id']} "
250
+ f"after {elapsed}s (task_timeout_sec={task_timeout_sec})",
251
+ flush=True,
252
+ )
253
+ return {
254
+ "task_id": s["task_id"],
255
+ "error": True,
256
+ "error_msg": f"per-task timeout after {elapsed}s",
257
+ "elapsed_sec": elapsed,
258
+ }
259
+ except Exception as e:
260
+ if verbose:
261
+ print(f"[{idx}/{len(samples)}] ROLLOUT FAILED {s['task_id']}: {e}",
262
+ flush=True)
263
+ traceback.print_exc()
264
+ return {"task_id": s["task_id"], "error": True, "error_msg": str(e)}
265
+
266
+ try:
267
+ if concurrency <= 1:
268
+ for i, s in enumerate(samples, 1):
269
+ per_task.append(await _run_one(i, s))
270
+ else:
271
+ sem = asyncio.Semaphore(concurrency)
272
+
273
+ async def _bound(i, s):
274
+ async with sem:
275
+ return await _run_one(i, s)
276
+
277
+ # return_exceptions=True: if one rollout raises, sibling rollouts
278
+ # keep running and we collect partial results. With False, gather
279
+ # cancels nothing on the first exception → the finally block then
280
+ # tears down MCP clients while in-flight rollouts are still using
281
+ # them, causing cascading crashes.
282
+ results = await asyncio.gather(
283
+ *(_bound(i, s) for i, s in enumerate(samples, 1)),
284
+ return_exceptions=True,
285
+ )
286
+ for i, r in enumerate(results, 1):
287
+ if isinstance(r, BaseException):
288
+ sid = samples[i - 1].get("task_id", f"sample-{i}")
289
+ if verbose:
290
+ print(f"[{i}/{len(samples)}] GATHER EXCEPTION {sid}: {r}",
291
+ flush=True)
292
+ per_task.append({"task_id": sid, "error": True,
293
+ "error_msg": str(r)})
294
+ else:
295
+ per_task.append(r)
296
+
297
+ # Aggregate JSONL after all rows are collected (avoids interleaved
298
+ # writes from concurrent rollouts).
299
+ if out_dir is not None:
300
+ with (out_dir / "trajectories.jsonl").open("w") as fh:
301
+ for r in per_task:
302
+ fh.write(json.dumps(r, ensure_ascii=False, default=str) + "\n")
303
+ finally:
304
+ # Cleanup MCP clients in the same task as init.
305
+ from slime_mcp_rollout.tasks import _REGISTRY
306
+ for h in _REGISTRY.values():
307
+ if hasattr(h, "cleanup"):
308
+ try:
309
+ await h.cleanup()
310
+ except Exception:
311
+ pass
312
+
313
+ elapsed_total = round(time.time() - started_at, 2)
314
+ n_pass = sum(1 for r in per_task if r.get("reward") == 1.0)
315
+ n_eval_err = sum(
316
+ 1 for r in per_task
317
+ if r.get("reward") != 1.0
318
+ and (r.get("meta") or {}).get("evaluator_internal_error")
319
+ )
320
+ n = len(per_task)
321
+ total_in = sum(r.get("in_tokens", 0) for r in per_task)
322
+ total_out = sum(r.get("out_tokens", 0) for r in per_task)
323
+ summary = {
324
+ "domain": domain,
325
+ "model": model_name,
326
+ "base_url": base_url,
327
+ "concurrency": concurrency,
328
+ "started_at": _dt.datetime.fromtimestamp(started_at).isoformat(),
329
+ "finished_at": _dt.datetime.now().isoformat(),
330
+ "elapsed_sec": elapsed_total,
331
+ "num_tasks": n,
332
+ "num_passed": n_pass,
333
+ "num_model_failed": n - n_pass - n_eval_err,
334
+ "num_evaluator_errors": n_eval_err,
335
+ "pass_rate_raw": round(n_pass / max(n, 1), 4),
336
+ "pass_rate_adjusted": (
337
+ "N/A (all failures were evaluator-side)"
338
+ if n - n_eval_err == 0
339
+ else round(n_pass / (n - n_eval_err), 4)
340
+ ),
341
+ "total_input_tokens": total_in,
342
+ "total_output_tokens": total_out,
343
+ "estimated_cost_usd_deepseek_v4_pro": round(
344
+ total_in / 1_000_000 * 0.435 + total_out / 1_000_000 * 0.87, 4
345
+ ),
346
+ "failed_tasks": [
347
+ {"task_id": r.get("task_id"),
348
+ "reasons": (r.get("meta") or {}).get("eval_reasons")}
349
+ for r in per_task
350
+ if r.get("reward") != 1.0
351
+ ],
352
+ }
353
+ if verbose:
354
+ print("=" * 70, flush=True)
355
+ print("SUMMARY", flush=True)
356
+ for k in ("num_tasks", "num_passed", "num_model_failed",
357
+ "num_evaluator_errors", "pass_rate_raw",
358
+ "pass_rate_adjusted", "elapsed_sec", "concurrency",
359
+ "total_input_tokens", "total_output_tokens",
360
+ "estimated_cost_usd_deepseek_v4_pro"):
361
+ print(f" {k}: {summary[k]}", flush=True)
362
+ if out_dir is not None:
363
+ (out_dir / "_summary.json").write_text(
364
+ json.dumps(summary, ensure_ascii=False, indent=2, default=str)
365
+ )
366
+ if verbose:
367
+ print(f"[synth] summary: {out_dir / '_summary.json'}", flush=True)
368
+ print(f"[synth] latest: {TRAJ_DIR / 'latest'}", flush=True)
369
+ return summary
370
+
371
+
372
+ # ---------------------------------------------------------------------------
373
+ # CLI
374
+ # ---------------------------------------------------------------------------
375
+ def _parse():
376
+ p = argparse.ArgumentParser(
377
+ description="Synthesize rollout trajectories from MCP-Universe tasks."
378
+ )
379
+ p.add_argument("--domain", default="financial_analysis",
380
+ help="MCP-Universe domain name (default: financial_analysis)")
381
+ p.add_argument("-n", "--num", type=int, default=1)
382
+ p.add_argument("--task-ids", nargs="*", default=None,
383
+ help="Run only these task_ids (overrides --num)")
384
+ p.add_argument("--collect-trace", action="store_true",
385
+ help="Write per-task trajectory JSON + aggregate jsonl")
386
+ p.add_argument("--concurrency", type=int, default=1,
387
+ help="N parallel rollouts via asyncio.gather "
388
+ "(each gets its own agent from the pool)")
389
+ p.add_argument("--task-timeout-sec", type=float, default=None,
390
+ help="Per-task wall-clock cap; stuck rollouts get cancelled "
391
+ "and logged as errors. Recommended for unattended runs.")
392
+ p.add_argument("--model", default=None,
393
+ help="Override model_name (default: deepseek-v4-pro)")
394
+ p.add_argument("--base-url", default=None,
395
+ help="Override base_url (default: DeepSeek)")
396
+ p.add_argument("--api-key", default=None,
397
+ help="Override api_key (default: env)")
398
+ return p.parse_args()
399
+
400
+
401
+ def main():
402
+ cli = _parse()
403
+ kwargs = dict(
404
+ domain=cli.domain,
405
+ num=cli.num,
406
+ task_ids=cli.task_ids,
407
+ collect_trace=cli.collect_trace,
408
+ concurrency=cli.concurrency,
409
+ task_timeout_sec=cli.task_timeout_sec,
410
+ )
411
+ if cli.model: kwargs["model_name"] = cli.model
412
+ if cli.base_url: kwargs["base_url"] = cli.base_url
413
+ if cli.api_key: kwargs["api_key"] = cli.api_key
414
+ asyncio.run(synthesize(**kwargs))
415
+
416
+
417
+ if __name__ == "__main__":
418
+ main()
slime_mcp_rollout/to_sft.py ADDED
@@ -0,0 +1,138 @@
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
1
+ """Convert rollout trajectories into an SFT dataset (OpenAI messages format).
2
+
3
+ Reads a rollout output dir (slime_mcp_rollout/data/trajectories/<run>/), keeps
4
+ only reward==1 trajectories, and emits one JSONL line per trajectory:
5
+
6
+ {
7
+ "task_id": ...,
8
+ "domain": "financial_analysis",
9
+ "template": "portfolio_return", # from the augment task meta
10
+ "reward": 1.0,
11
+ "messages": [ {role,content,tool_calls}, {role:tool,...}, ...,
12
+ {role:assistant, content:<final answer>} ],
13
+ "meta": {"trace_id", "in_tokens", "out_tokens", "elapsed_sec", "params"}
14
+ }
15
+
16
+ The message list is the agent's full tool-use conversation: we take the
17
+ longest LLM span's `messages` (the full history up to the final decision) and
18
+ append the agent's final answer as the closing assistant turn.
19
+
20
+ Usage:
21
+ python -m slime_mcp_rollout.to_sft --run-dir <dir> --out <file.jsonl>
22
+ """
23
+ from __future__ import annotations
24
+
25
+ import argparse
26
+ import glob
27
+ import json
28
+ import os
29
+ from pathlib import Path
30
+ from typing import Any, Dict, List, Optional
31
+
32
+ REPO_ROOT = Path(__file__).resolve().parents[1]
33
+ DEFAULT_DOMAIN = "browser_automation"
34
+
35
+
36
+ def _augment_dir(domain: str) -> Path:
37
+ return REPO_ROOT / "slime_mcp_rollout" / "augment_tasks" / domain
38
+
39
+
40
+ def _final_answer_text(task: Dict[str, Any]) -> str:
41
+ r = task.get("response")
42
+ if isinstance(r, str):
43
+ return r
44
+ return json.dumps(r, ensure_ascii=False)
45
+
46
+
47
+ def _longest_messages(trajectory: Any) -> Optional[List[Dict[str, Any]]]:
48
+ """Return the `messages` list from the LLM span that has the most messages
49
+ (i.e. the fullest conversation history)."""
50
+ best = None
51
+ if not isinstance(trajectory, list):
52
+ return None
53
+ for span in trajectory:
54
+ for rec in (span.get("records") or []):
55
+ d = rec.get("data") or {}
56
+ if d.get("type") == "llm":
57
+ msgs = d.get("messages")
58
+ if isinstance(msgs, list) and (best is None or len(msgs) > len(best)):
59
+ best = msgs
60
+ return best
61
+
62
+
63
+ def _template_for(task_id: str, domain: str) -> Optional[str]:
64
+ """Look up the augment template from the task JSON (task_id embeds run+idx)."""
65
+ name = task_id[len(domain) + 1:] if task_id.startswith(domain + "_") else task_id
66
+ p = _augment_dir(domain) / f"{name}.json"
67
+ if p.is_file():
68
+ try:
69
+ return json.load(open(p)).get("_augment_meta", {}).get("template")
70
+ except Exception: # pylint: disable=broad-exception-caught
71
+ return None
72
+ return None
73
+
74
+
75
+ def convert(run_dir: str, out_path: str, domain: str = DEFAULT_DOMAIN) -> Dict[str, int]:
76
+ files = [f for f in glob.glob(os.path.join(run_dir, "*.json"))
77
+ if not f.endswith("_summary.json")]
78
+ kept = 0
79
+ skipped_reward = 0
80
+ skipped_nomsg = 0
81
+ by_tpl: Dict[str, int] = {}
82
+ with open(out_path, "w") as out:
83
+ for f in files:
84
+ try:
85
+ task = json.load(open(f))
86
+ except Exception: # pylint: disable=broad-exception-caught
87
+ continue
88
+ if task.get("reward") != 1.0:
89
+ skipped_reward += 1
90
+ continue
91
+ msgs = _longest_messages(task.get("trajectory"))
92
+ if not msgs:
93
+ skipped_nomsg += 1
94
+ continue
95
+ messages = list(msgs)
96
+ messages.append({"role": "assistant", "content": _final_answer_text(task)})
97
+ tid = task.get("task_id", os.path.basename(f).replace(".json", ""))
98
+ tpl = _template_for(tid, domain)
99
+ by_tpl[tpl or "?"] = by_tpl.get(tpl or "?", 0) + 1
100
+ meta = task.get("meta", {}) or {}
101
+ record = {
102
+ "task_id": tid,
103
+ "domain": task.get("domain", domain),
104
+ "template": tpl,
105
+ "reward": 1.0,
106
+ "messages": messages,
107
+ "meta": {
108
+ "trace_id": meta.get("trace_id"),
109
+ "in_tokens": task.get("in_tokens"),
110
+ "out_tokens": task.get("out_tokens"),
111
+ "elapsed_sec": task.get("elapsed_sec"),
112
+ },
113
+ }
114
+ out.write(json.dumps(record, ensure_ascii=False) + "\n")
115
+ kept += 1
116
+ summary = {
117
+ "kept_reward1": kept,
118
+ "skipped_reward0": skipped_reward,
119
+ "skipped_no_messages": skipped_nomsg,
120
+ "by_template": by_tpl,
121
+ "out": out_path,
122
+ }
123
+ return summary
124
+
125
+
126
+ def main():
127
+ p = argparse.ArgumentParser()
128
+ p.add_argument("--run-dir", required=True, help="trajectories/<run> dir")
129
+ p.add_argument("--out", required=True, help="output .jsonl path")
130
+ p.add_argument("--domain", default=DEFAULT_DOMAIN,
131
+ help="domain label + augment_tasks/<domain>/ for template lookup")
132
+ args = p.parse_args()
133
+ s = convert(args.run_dir, args.out, args.domain)
134
+ print(json.dumps(s, indent=2, ensure_ascii=False))
135
+
136
+
137
+ if __name__ == "__main__":
138
+ main()
sqlc.yaml ADDED
@@ -0,0 +1,17 @@
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
1
+ version: "2"
2
+ plugins:
3
+ - name: py
4
+ wasm:
5
+ url: https://downloads.sqlc.dev/plugin/sqlc-gen-python_1.3.0.wasm
6
+ sha256: fbedae96b5ecae2380a70fb5b925fd4bff58a6cfb1f3140375d098fbab7b3a3c
7
+ sql:
8
+ - schema: "mcpuniverse/app/db/migration"
9
+ queries: "mcpuniverse/app/db/query"
10
+ engine: postgresql
11
+ codegen:
12
+ - out: "mcpuniverse/app/db/sqlc"
13
+ plugin: py
14
+ options:
15
+ package: "mcpuniverse.app.db.sqlc"
16
+ emit_sync_querier: true
17
+ emit_async_querier: true