langsmith_evals/experiments/claude.py

"""LangSmith experiment: full harmonic graph with Anthropic models.

Usage:
    LANGCHAIN_API_KEY=<key> ANTHROPIC_API_KEY=<key> \\
        python langsmith_evals/experiments/claude.py \\
        [--mcp-url http://localhost:7860/gradio_api/mcp/sse] \\
        [--analysis-model claude-sonnet-4-6] \\
        [--research-model claude-sonnet-4-6]
"""

import argparse
import sys
from pathlib import Path

sys.path.insert(0, str(Path(__file__).parent.parent.parent))

from dotenv import load_dotenv

load_dotenv()

from langchain_anthropic import ChatAnthropic
from langchain_community.tools import DuckDuckGoSearchRun

from langsmith_evals.evaluators import evaluate_formats_tool_called_once, evaluate_notable_songs
from langsmith_evals.runner import run
from langsmith_evals.targets import make_harmonic_target


def main():
    parser = argparse.ArgumentParser(description="Evaluate full harmonic graph with Anthropic models")
    parser.add_argument("--mcp-url", default="http://localhost:7860/gradio_api/mcp/sse")
    parser.add_argument("--analysis-model", default="claude-sonnet-4-6")
    parser.add_argument("--research-model", default="claude-sonnet-4-6")
    args = parser.parse_args()

    analysis_llm = ChatAnthropic(model=args.analysis_model)
    research_llm = ChatAnthropic(model=args.research_model)
    search_tools = [DuckDuckGoSearchRun()]

    run(
        make_harmonic_target(args.mcp_url, analysis_llm, research_llm, search_tools),
        prefix=f"harmonic-anthropic-{args.analysis_model}",
        evaluators=[evaluate_formats_tool_called_once, evaluate_notable_songs],
    )


if __name__ == "__main__":
    main()