"""Fixture data and tool loader for LangSmith eval runs. The similarity tools (``analyze_chord_sequence_text``, ``analyze_music_file``) are mocked with a ``DynamicMockMcpTool`` that honours the ``limit`` kwarg, slicing from a pool of 55 neighbours. This means the agent can request any number up to 55 and receive a correctly-sized response — enough to test that limits up to 50 are respected. Other tools (e.g. ``get_supported_chord_formats``) are fetched and called for real, so the schema the LLM sees always reflects the live server. """ import json import pathlib from agents.mcp import DynamicMockMcpTool, load_tools _FIXTURE_DIR = pathlib.Path(__file__).parent _SIMILARITY_POOL = json.loads((_FIXTURE_DIR / "similarity_fixture.json").read_text()) _SIMILARITY_SCORE = 0.38 # Tools whose invocation should be replaced with dynamic fixture data in eval runs. _DYNAMIC_MOCKED_TOOLS = {"analyze_chord_sequence_text", "analyze_music_file"} def load_eval_tools(mcp_url: str) -> list: """Load MCP tools from the server, replacing similarity tools with dynamic fixtures. Fetches all tool schemas live (so the LLM sees the real descriptions and parameter definitions), then substitutes ``DynamicMockMcpTool`` for any tool whose name appears in ``_DYNAMIC_MOCKED_TOOLS``. All other tools call the server normally. :param mcp_url: SSE endpoint of the running MCP server. """ tools = load_tools(mcp_url) return [ DynamicMockMcpTool( name=t.name, description=t.description, args_schema=t.args_schema, mcp_url=mcp_url, score=_SIMILARITY_SCORE, pool=_SIMILARITY_POOL, ) if t.name in _DYNAMIC_MOCKED_TOOLS else t for t in tools ]