langsmith_evals/fixtures.py

"""Fixture data and tool loader for LangSmith eval runs.

The similarity tools (``analyze_chord_sequence_text``, ``analyze_music_file``) are
mocked with a ``DynamicMockMcpTool`` that honours the ``limit`` kwarg, slicing from
a pool of 55 neighbours.  This means the agent can request any number up to 55 and
receive a correctly-sized response — enough to test that limits up to 50 are respected.

Other tools (e.g. ``get_supported_chord_formats``) are fetched and called for real,
so the schema the LLM sees always reflects the live server.
"""

import json
import pathlib

from agents.mcp import DynamicMockMcpTool, load_tools

_FIXTURE_DIR = pathlib.Path(__file__).parent
_SIMILARITY_POOL = json.loads((_FIXTURE_DIR / "similarity_fixture.json").read_text())
_SIMILARITY_SCORE = 0.38

# Tools whose invocation should be replaced with dynamic fixture data in eval runs.
_DYNAMIC_MOCKED_TOOLS = {"analyze_chord_sequence_text", "analyze_music_file"}


def load_eval_tools(mcp_url: str) -> list:
    """Load MCP tools from the server, replacing similarity tools with dynamic fixtures.

    Fetches all tool schemas live (so the LLM sees the real descriptions and
    parameter definitions), then substitutes ``DynamicMockMcpTool`` for any tool whose
    name appears in ``_DYNAMIC_MOCKED_TOOLS``.  All other tools call the server normally.

    :param mcp_url: SSE endpoint of the running MCP server.
    """
    tools = load_tools(mcp_url)
    return [
        DynamicMockMcpTool(
            name=t.name,
            description=t.description,
            args_schema=t.args_schema,
            mcp_url=mcp_url,
            score=_SIMILARITY_SCORE,
            pool=_SIMILARITY_POOL,
        )
        if t.name in _DYNAMIC_MOCKED_TOOLS
        else t
        for t in tools
    ]