Adapt fixture and runner

agents/claude.py

@@ -8,6 +8,12 @@ from .base import Agent, AgentResult, ToolCall
 
 _DEFAULT_MODEL = "claude-sonnet-4-6"
 
+_SYSTEM = (
+    "You are a harmonic analysis assistant. After analysing a chord sequence, present only "
+    "well-known results — songs that are recognisable hits or by widely-known artists. "
+    "Use your knowledge to filter out obscure tracks. If no well-known matches exist, say so."
+)
+
 
 class ClaudeAgent(Agent):
     """Agent using the Anthropic SDK, connecting to tools via an MCP SSE server."""
@@ -32,6 +38,7 @@ class ClaudeAgent(Agent):
                     response = self._client.messages.create(
                         model=self.model,
                         max_tokens=4096,
+                        system=_SYSTEM,
                         tools=tools,
                         messages=messages,
                     )
@@ -45,21 +52,23 @@ class ClaudeAgent(Agent):
                     tool_results = []
                     for block in response.content:
                         if block.type == "tool_use":
-                            mcp_result = await session.call_tool(block.name, dict(block.input))
-                            result_text = "\n".join(
-                                item.text for item in mcp_result.content if hasattr(item, "text")
-                            )
+                            result_text = await self._invoke_tool(session, block.name, dict(block.input))
                             tool_calls.append(ToolCall(name=block.name, args=dict(block.input), result=result_text))
                             tool_results.append({
                                 "type": "tool_result",
                                 "tool_use_id": block.id,
-                                "content": str(mcp_result.content),
+                                "content": result_text,
                             })
 
                     messages.append({"role": "assistant", "content": response.content})
                     messages.append({"role": "user", "content": tool_results})
 
 
+    async def _invoke_tool(self, session: ClientSession, name: str, args: dict) -> str:
+        mcp_result = await session.call_tool(name, args)
+        return "\n".join(item.text for item in mcp_result.content if hasattr(item, "text"))
+
+
 def _to_anthropic(tool) -> dict:
     return {
         "name": tool.name,

langsmith_evals/evaluators.py

@@ -22,11 +22,8 @@ def _count_tool_use_in_llm_runs(llm_runs: list[Run], tool_name: str) -> int:
     return count
 
 
-def evaluate_formats_tool_call_count(run: Run, example: Example) -> EvaluationResult:
-    """Count how many times the supported-formats tool was called.
-
-    Ideally the agent calls it exactly once before analysing chords.
-    A score of 0 means it was never called; higher values indicate redundant calls.
+def evaluate_formats_tool_called_once(run: Run, example: Example) -> EvaluationResult:
+    """Check that the supported-formats tool was called exactly once.
 
     :param run: LangSmith run — LLM child runs are fetched via the client.
     :param example: Dataset example (unused).
@@ -35,12 +32,59 @@ def evaluate_formats_tool_call_count(run: Run, example: Example) -> EvaluationRe
     llm_runs = list(client.list_runs(trace_id=run.id, run_type="llm"))
     count = _count_tool_use_in_llm_runs(llm_runs, _FORMATS_TOOL)
     return EvaluationResult(
-        key="formats_tool_call_count",
-        score=count,
+        key="formats_tool_called_once",
+        score=int(count == 1),
         comment=f"{_FORMATS_TOOL} called {count} time(s)",
     )
 
 
+def _title_in_text(title: str, text: str) -> bool:
+    return title.lower() in text.lower()
+
+
+def evaluate_notable_songs_recalled(run: Run, example: Example) -> EvaluationResult:
+    """Fraction of expected notable songs that appear in the agent's response.
+
+    1.0 = all present (green), 0.0 = none present (red).
+
+    :param run: LangSmith run — expects ``outputs["response"]`` as a string.
+    :param example: Dataset example — expects ``outputs["expected_in_response"]`` as a list of titles.
+    """
+    response = (run.outputs or {}).get("response", "")
+    expected = (example.outputs or {}).get("expected_in_response", [])
+    if not expected:
+        return EvaluationResult(key="notable_songs_recalled", score=None,
+                                comment="no expected_in_response in example outputs")
+    found = [t for t in expected if _title_in_text(t, response)]
+    return EvaluationResult(
+        key="notable_songs_recalled",
+        score=len(found) / len(expected),
+        comment=f"{len(found)}/{len(expected)} notable songs present in response",
+    )
+
+
+def evaluate_non_notable_songs_excluded(run: Run, example: Example) -> EvaluationResult:
+    """Fraction of expected-excluded songs that do NOT appear in the agent's response.
+
+    1.0 = none leaked (green), 0.0 = all leaked (red).
+
+    :param run: LangSmith run — expects ``outputs["response"]`` as a string.
+    :param example: Dataset example — expects ``outputs["expected_not_in_response"]`` as a list of titles.
+    """
+    response = (run.outputs or {}).get("response", "")
+    excluded = (example.outputs or {}).get("expected_not_in_response", [])
+    if not excluded:
+        return EvaluationResult(key="non_notable_songs_excluded", score=None,
+                                comment="no expected_not_in_response in example outputs")
+    leaked = [t for t in excluded if _title_in_text(t, response)]
+    return EvaluationResult(
+        key="non_notable_songs_excluded",
+        score=1 - len(leaked) / len(excluded),
+        comment=(f"{len(leaked)} excluded song(s) found: {leaked}" if leaked
+                 else "no excluded songs in response"),
+    )
+
+
 def evaluate_result_count(run: Run, example: Example) -> EvaluationResult:
     """Check the number of returned songs does not exceed the requested limit.
 

langsmith_evals/runner.py

@@ -4,19 +4,23 @@ from langsmith import Client
 from langsmith import evaluate as ls_evaluate
 
 from langsmith_evals.dataset import DATASET_NAME
-from langsmith_evals.evaluators import evaluate_formats_tool_call_count, evaluate_result_count
+from langsmith_evals.evaluators import evaluate_formats_tool_called_once, evaluate_result_count
 
+_DEFAULT_EVALUATORS = [evaluate_result_count, evaluate_formats_tool_called_once]
 
-def run(target, prefix: str) -> None:
-    """Run a LangSmith experiment against the harmonic analysis dataset.
+
+def run(target, prefix: str, evaluators: list | None = None, dataset: str | None = None) -> None:
+    """Run a LangSmith experiment against a dataset.
 
     :param target: Callable ``(inputs: dict) -> dict`` to evaluate.
     :param prefix: Experiment name prefix shown in the LangSmith UI.
+    :param evaluators: Evaluator functions. Defaults to result-count + formats-tool-called-once.
+    :param dataset: Dataset name. Defaults to the main chord-sequences dataset.
     """
     ls_evaluate(
         target,
-        data=DATASET_NAME,
-        evaluators=[evaluate_result_count, evaluate_formats_tool_call_count],
+        data=dataset or DATASET_NAME,
+        evaluators=evaluators or _DEFAULT_EVALUATORS,
         experiment_prefix=prefix,
         client=Client(),
     )

langsmith_evals/similarity_fixture.json

@@ -1,57 +1,59 @@
 [
   {"title": "Let It Be", "artist": "The Beatles", "similarity": 0.97},
-  {"title": "No Woman No Cry", "artist": "Bob Marley", "similarity": 0.961},
-  {"title": "Africa", "artist": "Toto", "similarity": 0.952},
+  {"title": "Machine Hit", "artist": "John Harald", "similarity": 0.718},
+  {"title": "Ho Hey", "artist": "The Lumineers", "similarity": 0.718},
   {"title": "With or Without You", "artist": "U2", "similarity": 0.943},
+  {"title": "Spring Song", "artist": "Gryphon", "similarity": 0.709},
   {"title": "Stand By Me", "artist": "Ben E. King", "similarity": 0.934},
-  {"title": "Zombie", "artist": "The Cranberries", "similarity": 0.925},
-  {"title": "Knockin on Heavens Door", "artist": "Bob Dylan", "similarity": 0.916},
-  {"title": "Wonderful Tonight", "artist": "Eric Clapton", "similarity": 0.907},
-  {"title": "Brown Eyed Girl", "artist": "Van Morrison", "similarity": 0.898},
-  {"title": "Take Me Home Country Roads", "artist": "John Denver", "similarity": 0.889},
+  {"title": "Walter Murphy and the Big Apple Band", "artist": "A Fifth of Beethoven", "similarity": 0.934},
+  {"title": "My Life", "artist": "Oliver Holloway", "similarity": 0.736},
   {"title": "Someone Like You", "artist": "Adele", "similarity": 0.880},
-  {"title": "Counting Stars", "artist": "OneRepublic", "similarity": 0.871},
+  {"title": "Little Talks", "artist": "Of Monsters and Men", "similarity": 0.700},
+  {"title": "Wonderwall", "artist": "Oasis", "similarity": 0.790},
   {"title": "Wagon Wheel", "artist": "Old Crow Medicine Show", "similarity": 0.862},
-  {"title": "I'm Yours", "artist": "Jason Mraz", "similarity": 0.853},
-  {"title": "Hey Soul Sister", "artist": "Train", "similarity": 0.844},
-  {"title": "Photograph", "artist": "Ed Sheeran", "similarity": 0.835},
-  {"title": "Perfect", "artist": "Ed Sheeran", "similarity": 0.826},
   {"title": "Can't Help Falling in Love", "artist": "Elvis Presley", "similarity": 0.817},
+  {"title": "Riptide", "artist": "Vance Joy", "similarity": 0.727},
   {"title": "Hallelujah", "artist": "Leonard Cohen", "similarity": 0.808},
-  {"title": "Horse With No Name", "artist": "America", "similarity": 0.799},
-  {"title": "Wonderwall", "artist": "Oasis", "similarity": 0.790},
-  {"title": "Don't Look Back in Anger", "artist": "Oasis", "similarity": 0.781},
-  {"title": "21 Guns", "artist": "Green Day", "similarity": 0.772},
   {"title": "Save Tonight", "artist": "Eagle-Eye Cherry", "similarity": 0.763},
+  {"title": "No Woman No Cry", "artist": "Bob Marley", "similarity": 0.961},
   {"title": "Mr Jones", "artist": "Counting Crows", "similarity": 0.754},
+  {"title": "Piano Man", "artist": "Billy Joel", "similarity": 0.502},
+  {"title": "Hey Soul Sister", "artist": "Train", "similarity": 0.844},
+  {"title": "Hotel California", "artist": "Eagles", "similarity": 0.493},
+  {"title": "Horse With No Name", "artist": "America", "similarity": 0.799},
+  {"title": "Africa", "artist": "Toto", "similarity": 0.952},
+  {"title": "More Than Words", "artist": "Extreme", "similarity": 0.682},
+  {"title": "Zombie", "artist": "The Cranberries", "similarity": 0.925},
+  {"title": "Let Her Go", "artist": "Passenger", "similarity": 0.529},
+  {"title": "Knockin on Heavens Door", "artist": "Bob Dylan", "similarity": 0.916},
+  {"title": "What's Up", "artist": "4 Non Blondes", "similarity": 0.556},
+  {"title": "Wonderful Tonight", "artist": "Eric Clapton", "similarity": 0.907},
+  {"title": "The Lazy Song", "artist": "Bruno Mars", "similarity": 0.592},
+  {"title": "Brown Eyed Girl", "artist": "Van Morrison", "similarity": 0.898},
   {"title": "Good Riddance", "artist": "Green Day", "similarity": 0.745},
-  {"title": "I Will Follow You Into the Dark", "artist": "Death Cab for Cutie", "similarity": 0.736},
-  {"title": "Riptide", "artist": "Vance Joy", "similarity": 0.727},
-  {"title": "Ho Hey", "artist": "The Lumineers", "similarity": 0.718},
-  {"title": "Home", "artist": "Phillip Phillips", "similarity": 0.709},
-  {"title": "Little Talks", "artist": "Of Monsters and Men", "similarity": 0.700},
+  {"title": "Take Me Home Country Roads", "artist": "John Denver", "similarity": 0.889},
   {"title": "Iris", "artist": "Goo Goo Dolls", "similarity": 0.691},
-  {"title": "More Than Words", "artist": "Extreme", "similarity": 0.682},
+  {"title": "Counting Stars", "artist": "OneRepublic", "similarity": 0.871},
   {"title": "Tears in Heaven", "artist": "Eric Clapton", "similarity": 0.673},
+  {"title": "I'm Yours", "artist": "Jason Mraz", "similarity": 0.853},
   {"title": "The Scientist", "artist": "Coldplay", "similarity": 0.664},
+  {"title": "Photograph", "artist": "Ed Sheeran", "similarity": 0.835},
   {"title": "Yellow", "artist": "Coldplay", "similarity": 0.655},
+  {"title": "Perfect", "artist": "Ed Sheeran", "similarity": 0.826},
   {"title": "Fix You", "artist": "Coldplay", "similarity": 0.646},
+  {"title": "Don't Look Back in Anger", "artist": "Oasis", "similarity": 0.781},
   {"title": "Thinking Out Loud", "artist": "Ed Sheeran", "similarity": 0.637},
+  {"title": "21 Guns", "artist": "Green Day", "similarity": 0.772},
   {"title": "Love Yourself", "artist": "Justin Bieber", "similarity": 0.628},
-  {"title": "Despacito", "artist": "Luis Fonsi", "similarity": 0.619},
   {"title": "Shape of You", "artist": "Ed Sheeran", "similarity": 0.610},
+  {"title": "Despacito", "artist": "Luis Fonsi", "similarity": 0.619},
   {"title": "Just the Way You Are", "artist": "Bruno Mars", "similarity": 0.601},
-  {"title": "The Lazy Song", "artist": "Bruno Mars", "similarity": 0.592},
   {"title": "Grenade", "artist": "Bruno Mars", "similarity": 0.583},
   {"title": "Viva la Vida", "artist": "Coldplay", "similarity": 0.574},
   {"title": "Under the Bridge", "artist": "Red Hot Chili Peppers", "similarity": 0.565},
-  {"title": "What's Up", "artist": "4 Non Blondes", "similarity": 0.556},
   {"title": "La Bamba", "artist": "Ritchie Valens", "similarity": 0.547},
   {"title": "Shallow", "artist": "Lady Gaga and Bradley Cooper", "similarity": 0.538},
-  {"title": "Let Her Go", "artist": "Passenger", "similarity": 0.529},
   {"title": "Fast Car", "artist": "Tracy Chapman", "similarity": 0.520},
   {"title": "Blowin in the Wind", "artist": "Bob Dylan", "similarity": 0.511},
-  {"title": "Piano Man", "artist": "Billy Joel", "similarity": 0.502},
-  {"title": "Hotel California", "artist": "Eagles", "similarity": 0.493},
   {"title": "Losing My Religion", "artist": "R.E.M.", "similarity": 0.484}
 ]