Spaces:
Sleeping
Sleeping
| """Target factory functions for LangSmith experiments. | |
| A target is a ``(inputs: dict) -> dict`` callable passed to ``ls_evaluate``. | |
| Each factory builds and closes over the agent so it is constructed once and | |
| shared across all dataset examples in a run. | |
| """ | |
| import asyncio | |
| import json | |
| import anthropic as anthropic_sdk | |
| from langchain_core.language_models import BaseChatModel | |
| from langchain_core.tools import BaseTool | |
| from langsmith.wrappers import wrap_anthropic | |
| from mcp import ClientSession | |
| from mcp.client.sse import sse_client | |
| from agents.multi import build_harmonic_graph | |
| from langsmith_evals.fixtures import ( | |
| _DYNAMIC_MOCKED_TOOLS, | |
| _SIMILARITY_POOL, | |
| _SIMILARITY_SCORE, | |
| load_eval_tools, | |
| ) | |
| _SYSTEM = ( | |
| "You are a harmonic analysis assistant. After analysing a chord sequence, present only " | |
| "well-known results — songs that are recognisable hits or by widely-known artists. " | |
| "Use your knowledge to filter out obscure tracks. If no well-known matches exist, say so." | |
| ) | |
| def make_harmonic_target( | |
| mcp_url: str, | |
| analysis_llm: BaseChatModel, | |
| research_llm: BaseChatModel, | |
| search_tools: list[BaseTool], | |
| ) -> callable: | |
| """Target wrapping the full harmonic graph. | |
| Analysis similarity tools are mocked with fixture data so runs are | |
| deterministic and independent of the live FAISS index. Research uses | |
| the provided search tools, which may be live or mocked. | |
| :param mcp_url: SSE endpoint of the running MCP server. | |
| :param analysis_llm: LLM for the analysis subgraph. | |
| :param research_llm: LLM for the research subgraph. | |
| :param search_tools: Web search tools for the research subgraph. | |
| """ | |
| mcp_tools = load_eval_tools(mcp_url) | |
| graph = build_harmonic_graph(analysis_llm, research_llm, mcp_tools, search_tools) | |
| def target(inputs: dict) -> dict: | |
| user_input = inputs["user_input"] | |
| print(f"\n>>> input: {user_input!r}") | |
| response = "" | |
| for ns, chunk in graph.stream({"user_input": user_input}, stream_mode="updates", subgraphs=True): | |
| for node_name, update in chunk.items(): | |
| prefix = f"{ns[-1].split(':')[0]}/" if ns else "" | |
| print(f" [node] {prefix}{node_name}") | |
| if not ns and "response" in update: | |
| response = update["response"] | |
| print(f"<<< response: {response!r}") | |
| return {"response": response} | |
| return target | |
| def make_claude_sdk_target(mcp_url: str, model: str) -> callable: | |
| """Target running the Anthropic SDK directly with native MCP. | |
| Uses ``wrap_anthropic`` so every API call is captured in LangSmith. | |
| Similarity tools are mocked with the same fixture pool used by the | |
| LangGraph experiments; all other tools call the live server. | |
| :param mcp_url: SSE endpoint of the running MCP server. | |
| :param model: Anthropic model ID. | |
| """ | |
| sdk_client = wrap_anthropic(anthropic_sdk.Anthropic()) | |
| async def _run(user_input: str) -> str: | |
| async with sse_client(mcp_url) as (read, write): | |
| async with ClientSession(read, write) as session: | |
| await session.initialize() | |
| tools = [ | |
| {"name": t.name, "description": t.description or "", "input_schema": t.inputSchema} | |
| for t in (await session.list_tools()).tools | |
| ] | |
| messages = [{"role": "user", "content": user_input}] | |
| while True: | |
| response = sdk_client.messages.create( | |
| model=model, | |
| max_tokens=4096, | |
| system=_SYSTEM, | |
| tools=tools, | |
| messages=messages, | |
| ) | |
| if response.stop_reason == "end_turn": | |
| return next((b.text for b in response.content if b.type == "text"), "") | |
| tool_results = [] | |
| for block in response.content: | |
| if block.type != "tool_use": | |
| continue | |
| args = dict(block.input) | |
| if block.name in _DYNAMIC_MOCKED_TOOLS: | |
| limit = int(args.get("limit", len(_SIMILARITY_POOL))) | |
| result_text = f"{_SIMILARITY_SCORE}\n{json.dumps(_SIMILARITY_POOL[:limit])}" | |
| else: | |
| mcp_result = await session.call_tool(block.name, args) | |
| result_text = "\n".join( | |
| item.text for item in mcp_result.content if hasattr(item, "text") | |
| ) | |
| print(f" [tool] {block.name}({args})") | |
| tool_results.append({ | |
| "type": "tool_result", | |
| "tool_use_id": block.id, | |
| "content": result_text, | |
| }) | |
| messages.append({"role": "assistant", "content": response.content}) | |
| messages.append({"role": "user", "content": tool_results}) | |
| def target(inputs: dict) -> dict: | |
| user_input = inputs["user_input"] | |
| print(f"\n>>> input: {user_input!r}") | |
| response = asyncio.run(_run(user_input)) | |
| print(f"<<< response: {response!r}") | |
| return {"response": response} | |
| return target | |