Strands-Agents SDK Evaluators

evals/README.md

@@ -11,7 +11,6 @@ evals/
 ├── task_functions.py        # Agent wrapper functions
 ├── langfuse_reporter.py     # Langfuse integration
 ├── dataset_loader.py        # JSON to Case conversion
-├── generate_experiment.py   # SDK Experiment Generator
 ├── run_helpfulness.py       # Helpfulness evaluation
 ├── run_trajectory.py        # Trajectory evaluation
 └── run_full_suite.py        # Full evaluation suite
@@ -36,12 +35,6 @@ python -m evals.run_trajectory
 python -m evals.run_full_suite
 ```
 
-### Generate Synthetic Data
-
-```bash
-# Generate new test cases using SDK ExperimentGenerator
-python -m evals.generate_experiment
-```
 ## Features
 
 - **SDK-Aligned**: Uses `Experiment.run_evaluations()` framework

evals/config.py

@@ -2,7 +2,7 @@
 Shared configuration for evaluators and models.
 """
 from strands.models.openai import OpenAIModel
-from strands_evals.evaluators import OutputEvaluator, TrajectoryEvaluator, FaithfulnessEvaluator
+from strands_evals.evaluators import OutputEvaluator, TrajectoryEvaluator
 
 # Configure OpenAI model for evaluators
 eval_model = OpenAIModel(model_id="gpt-4o")
@@ -23,7 +23,7 @@ helpfulness_evaluator = OutputEvaluator(
     model=eval_model
 )
 
-# Faithfulness Evaluator (Generic for now, to enable data generation)
+# Faithfulness Evaluator
 faithfulness_evaluator = OutputEvaluator(
     rubric="""
     Evaluate if the response is faithful to the retrieved context.
@@ -43,17 +43,13 @@ trajectory_evaluator = TrajectoryEvaluator(
     rubric="""
     Evaluate the tool usage trajectory:
     1. Correct tool selection - Were the right tools chosen?
-    2. Proper sequence - Logical order (Retrieve -> Analyze -> Explain)?
+    2. Proper sequence - Logical order (Retrieve → Analyze → Explain)?
     3. Efficiency - No unnecessary tools?
     
     Score 1.0 if optimal tools used correctly.
     Score 0.5 if correct tools but suboptimal sequence.
-    SCORE 0.0 if wrong tools or major inefficiencies.
+    Score 0.0 if wrong tools or major inefficiencies.
     """,
     include_inputs=True,
     model=eval_model
 )
-
-# Key Points Evaluator (Custom)
-from evals.key_points_evaluator import KeyPointsEvaluator
-key_points_evaluator = KeyPointsEvaluator()

evals/generate_experiment.py

@@ -1,86 +0,0 @@
-#!/usr/bin/env python
-"""
-Generate synthetic test cases using Strands ExperimentGenerator.
-
-This script replaces the custom generate_data.py and uses the SDK
-to generate diverse, high-quality test cases for the fraud agent.
-"""
-import os
-import json
-import asyncio
-from dotenv import load_dotenv
-
-load_dotenv()
-
-from typing import List, Dict
-from strands_evals.generators import ExperimentGenerator
-from strands_evals.evaluators import OutputEvaluator
-from evals.config import eval_model
-
-# Context description for the generator
-CONTEXT = """
-You are generating test cases for a Fraud Model Explainability Assistant for a financial services company.
-The assistant uses RAG and tools to explain fraud scores (0-1000), SHAP values, and compliance checks.
-
-Users are typically:
-1. Fraud Analysts (investigating specific cases)
-2. Data Scientists (monitoring model performance)
-3. Compliance Officers (checking for Fair Lending bias)
-4. Executives (asking for high-level summaries)
-
-Tools available:
-- get_application_summary(app_id): Returns score, risk level.
-- explain_fraud_score(app_id): Returns SHAP feature contributions.
-- compare_to_population(app_id): Returns stats vs approved/denied.
-- check_fair_lending_flags(app_id): Returns bias analysis.
-- get_identity_network(app_id): Returns linked applications.
-"""
-
-async def generate():
-    print("🚀 Starting Experiment Generation with SDK...")
-    
-    # Initialize generator with str input/output
-    generator = ExperimentGenerator[str, str](
-        input_type=str,
-        output_type=str,
-        model=eval_model
-    )
-    
-    # Generate experiment
-    print("   Generating cases (this may take a minute)...")
-    experiment = await generator.from_context_async(
-        context=CONTEXT,
-        num_cases=10,  # Generate 10 new cases
-        evaluator=OutputEvaluator, # Pass class, let generator create rubric
-        task_description="Explain fraud model decisions and risk factors.",
-        num_topics=5   # Split across different topics (High Risk, Compliance, etc.)
-    )
-    
-    print(f"✅ Generated {len(experiment.cases)} new test cases.")
-    
-    # Convert to our JSON format
-    new_cases = []
-    for i, case in enumerate(experiment.cases):
-        # Metadata might be None
-        metadata = case.metadata if case.metadata else {}
-        new_case = {
-            "id": f"synth_sdk_{i+1}",
-            "question": case.input,
-            "expected_intent": metadata.get("topic", "General"),
-            "expected_answer_key_points": [case.expected_output] if case.expected_output else []
-        }
-        new_cases.append(new_case)
-        print(f"   - [{new_case['expected_intent']}] {new_case['question'][:60]}...")
-
-    # Load existing cases to append (optional, or overwrite)
-    output_path = "evaluation/dataset_sdk.json"
-    
-    # Saving to a new file to avoid overwriting the main dataset during this test
-    with open(output_path, "w") as f:
-        json.dump(new_cases, f, indent=2)
-        
-    print(f"\n💾 Saved {len(new_cases)} cases to {output_path}")
-    print("   Review the file and merge into evaluation/dataset.json if desired.")
-
-if __name__ == "__main__":
-    asyncio.run(generate())

evals/key_points_evaluator.py

@@ -1,102 +0,0 @@
-"""
-Custom evaluator for checking if specific key points are present in the response.
-"""
-from typing import Any, List
-from strands_evals.evaluators import Evaluator
-from strands_evals.types.evaluation import EvaluationData, EvaluationOutput
-from strands_evals.types.trace import EvaluationLevel
-from typing_extensions import TypeVar
-
-InputT = TypeVar("InputT")
-OutputT = TypeVar("OutputT")
-
-class KeyPointsEvaluator(Evaluator[InputT, OutputT]):
-    """Evaluates output by checking for presence of expected key points (keywords/phrases)."""
-    
-    evaluation_level = EvaluationLevel.TRACE_LEVEL
-
-    def __init__(self, version: str = "v1"):
-        super().__init__()
-        self.version = version
-
-    def evaluate(self, evaluation_case: EvaluationData[InputT, OutputT]) -> List[EvaluationOutput]:
-        """Synchronous evaluation."""
-        return self._do_evaluation(evaluation_case)
-
-    async def evaluate_async(self, evaluation_case: EvaluationData[InputT, OutputT]) -> List[EvaluationOutput]:
-        """Asynchronous evaluation."""
-        return self._do_evaluation(evaluation_case)
-
-    def _do_evaluation(self, evaluation_case: EvaluationData[InputT, OutputT]) -> List[EvaluationOutput]:
-        """
-        Check if expected key points are present in the actual output.
-        Expects 'expected_key_points' list in case metadata.
-        """
-        # Get actual output
-        actual_output = str(evaluation_case.actual_output)
-        
-        # Get expectations from case metadata (which is attached to evaluation_case)
-        # Note: The SDK passes the whole Case object or relevant parts.
-        # However, EvaluationData typically has input/output. 
-        # Metadata is likely accessible if evaluation_case is constructed from a Case.
-        # But SDK EvaluationData doesn't strictly carry metadata field in all versions.
-        # We rely on how Experiment constructs it.
-        
-        # EXPERIMENTAL: The SDK's Experiment loop constructs EvaluationData.
-        # If it doesn't pass metadata, we need to inspect the source 'case'. 
-        # But Evaluator.evaluate receives EvaluationData, not Case.
-        # Wait, Strands SDK 1.22 might have metadata on EvaluationData?
-        # Let's check the type definition if needed. 
-        # For now, assuming we can access it or we need a workaround.
-        
-        # Workaround: For this custom evaluator to work with Experiment, 
-        # the Experiment must pass metadata. 
-        
-        # Actually, looking at the Experiment source (which we can't see right now but inferred),
-        # it might be easier to pass expected_output as the key points string?
-        # Dataset loader sets: expected_key_points in metadata.
-        
-        # Let's try to access metadata if it exists on EvaluationData, 
-        # Otherwise fall back to a safe default.
-        
-        key_points = []
-        if hasattr(evaluation_case, 'metadata') and evaluation_case.metadata:
-            key_points = evaluation_case.metadata.get("expected_key_points", [])
-        
-        # Calculate score
-        if not key_points:
-            return [EvaluationOutput(
-                score=1.0, 
-                test_pass=True, 
-                reason="No key points defined for this case.",
-                label="N/A"
-            )]
-
-        hits = 0
-        misses = []
-        
-        for point in key_points:
-            point_lower = point.lower()
-            output_lower = actual_output.lower()
-            
-            if point_lower in output_lower:
-                hits += 1
-            # partial match check (heuristic from run_full_suite)
-            elif any(word in output_lower for word in point_lower.split() if len(word) > 4):
-                 hits += 0.5
-                 misses.append(f"{point} (Partial)")
-            else:
-                 misses.append(point)
-
-        score = min(1.0, hits / len(key_points))
-        
-        reason = f"Matched {hits}/{len(key_points)} key points."
-        if misses:
-            reason += f" Missed: {', '.join(misses[:3])}..."
-            
-        return [EvaluationOutput(
-            score=score,
-            test_pass=score >= 0.7, # 70% threshold
-            reason=reason,
-            label=f"{int(score*100)}%"
-        )]

evals/langfuse_reporter.py

@@ -58,14 +58,13 @@ class LangfuseReporter:
         self.lf_client = langfuse_client
         self.tracer = trace.get_tracer("evaluation_reporter")
     
-    def log_experiment_results(self, reports, case_names: List[str], evaluator_names: List[str] = None) -> Dict[str, str]:
+    def log_experiment_results(self, reports, case_names: List[str]) -> Dict[str, str]:
         """
         Log SDK experiment results to Langfuse with OpenTelemetry trace correlation.
         
         Args:
             reports: List of evaluation reports from Experiment.run_evaluations()
             case_names: List of case names to create trace IDs for
-            evaluator_names: Optional list of evaluator names. If not provided, generic names will be used.
             
         Returns:
             Dict mapping case names to trace IDs
@@ -81,16 +80,14 @@ class LangfuseReporter:
                 trace_ids[case_name] = trace_id_hex
                 
                 # Log all evaluator scores for this case
-                for j, report in enumerate(reports):
-                    if i < len(report.scores):
-                        eval_name = evaluator_names[j] if evaluator_names and j < len(evaluator_names) else f"Evaluator_{j}"
-                        eval_name = eval_name.replace("Evaluator", "")
-                        
+                for report in reports:
+                    if i < len(report.case_results):
+                        case_result = report.case_results[i]
                         self.lf_client.score_trace(
                             trace_id=trace_id_hex,
-                            name=eval_name,
-                            value=report.scores[i],
-                            comment=report.reasons[i] if i < len(report.reasons) else None
+                            name=type(report.evaluator).__name__.replace("Evaluator", ""),
+                            value=case_result.evaluation_output.score,
+                            comment=case_result.evaluation_output.reason
                         )
         
         return trace_ids

evals/run_full_suite.py

@@ -8,11 +8,10 @@ and logs results to Langfuse with OpenTelemetry trace correlation.
 import json
 from strands_evals import Experiment
 from strands_evals.types.evaluation import EvaluationData
-from evals.config import helpfulness_evaluator, faithfulness_evaluator, trajectory_evaluator, key_points_evaluator
+from evals.config import helpfulness_evaluator, faithfulness_evaluator, trajectory_evaluator
 from evals.task_functions import get_fraud_explanation_with_trace
 from evals.dataset_loader import load_cases_from_json
 from evals.langfuse_reporter import LangfuseClient, LangfuseReporter
-from evals.utils import get_report_summary
 
 
 def main():
@@ -36,8 +35,7 @@ def main():
         evaluators=[
             helpfulness_evaluator,
             faithfulness_evaluator,
-            trajectory_evaluator,
-            key_points_evaluator
+            trajectory_evaluator
         ]
     )
     
@@ -49,11 +47,11 @@ def main():
     print("="*60 + "\n")
     
     for i, report in enumerate(reports):
-        evaluator_name = type(experiment.evaluators[i]).__name__
+        evaluator_name = type(report.evaluator).__name__
         print(f"\n### {evaluator_name} ###")
-        report.display()
+        report.run_display()
         
-        summary = get_report_summary(report)
+        summary = report.get_summary()
         print(f"\n📊 {evaluator_name} Summary:")
         print(f"   Pass Rate: {summary['pass_rate']:.1%}")
         print(f"   Average Score: {summary['average_score']:.2f}")
@@ -63,12 +61,47 @@ def main():
     print("\n" + "="*60)
     print("📤 Logging to Langfuse...")
     case_names = [case.name for case in cases]
-    evaluator_names = [type(e).__name__ for e in experiment.evaluators]
-    trace_ids = reporter.log_experiment_results(reports, case_names, evaluator_names)
+    trace_ids = reporter.log_experiment_results(reports, case_names)
     print(f"   ✅ Logged {len(trace_ids)} traces with {len(reports)} metrics each")
     
+    # Add Goal Success (heuristic evaluator)
+    print("\n🎯 Running Goal Success (heuristic)...")
+    goal_success_scores = []
+    
+    for case in cases:
+        expected_key_points = case.metadata.get("expected_key_points", [])
+        
+        # Get the answer for this case (from first report)
+        case_idx = cases.index(case)
+        answer = reports[0].case_results[case_idx].actual_output
+        
+        # Calculate hits
+        hits = 0
+        if expected_key_points:
+            for point in expected_key_points:
+                if point.lower() in answer.lower():
+                    hits += 1
+                elif any(word in answer.lower() for word in point.split() if len(word) > 4):
+                    hits += 0.5
+            success_rate = min(1.0, hits / len(expected_key_points))
+        else:
+            success_rate = 1.0
+        
+        goal_success_scores.append(success_rate)
+        
+        # Log to Langfuse
+        lf_client.score_trace(
+            trace_id=trace_ids[case.name],
+            name="Goal Success",
+            value=success_rate,
+            comment=f"Matched {hits}/{len(expected_key_points)} key points"
+        )
+    
+    avg_goal_success = sum(goal_success_scores) / len(goal_success_scores)
+    print(f"   Average Goal Success: {avg_goal_success:.2f}")
+    
     # Save experiment
-    experiment.to_file("experiment_files/full_suite_eval")
+    experiment.to_file("full_suite_eval", "json")
     print("\n💾 Experiment saved to ./experiment_files/full_suite_eval.json")
     
     # Final summary
@@ -76,7 +109,7 @@ def main():
     print("✅ EVALUATION COMPLETE")
     print("="*60)
     print(f"\nTotal Cases: {len(cases)}")
-    print(f"Evaluators: {len(reports)}")
+    print(f"Evaluators: {len(reports) + 1} (including Goal Success)")
     print(f"Langfuse Traces: {len(trace_ids)}")
     print(f"\nView results in Langfuse dashboard")
 

evals/run_helpfulness.py

@@ -10,7 +10,6 @@ from evals.config import helpfulness_evaluator
 from evals.task_functions import get_fraud_explanation
 from evals.dataset_loader import load_cases_from_json
 from evals.langfuse_reporter import LangfuseClient, LangfuseReporter
-from evals.utils import get_report_summary
 
 
 def main():
@@ -32,10 +31,10 @@ def main():
     
     # Display SDK results
     print("\n" + "="*60)
-    reports[0].display()
+    reports[0].run_display()
     
     # Get summary
-    summary = get_report_summary(reports[0])
+    summary = reports[0].get_summary()
     print(f"\n📊 Summary:")
     print(f"   Pass Rate: {summary['pass_rate']:.1%}")
     print(f"   Average Score: {summary['average_score']:.2f}")
@@ -45,12 +44,11 @@ def main():
     lf_client = LangfuseClient()
     reporter = LangfuseReporter(lf_client)
     case_names = [case.name for case in cases]
-    evaluator_names = [type(e).__name__ for e in experiment.evaluators]
-    trace_ids = reporter.log_experiment_results(reports, case_names, evaluator_names)
+    trace_ids = reporter.log_experiment_results(reports, case_names)
     print(f"   Logged {len(trace_ids)} traces to Langfuse")
     
     # Save experiment
-    experiment.to_file("experiment_files/helpfulness_eval")
+    experiment.to_file("helpfulness_eval", "json")
     print("\n💾 Experiment saved to ./experiment_files/helpfulness_eval.json")
 
 

evals/run_trajectory.py

@@ -9,7 +9,6 @@ from evals.config import trajectory_evaluator
 from evals.task_functions import get_fraud_explanation_with_trace
 from evals.dataset_loader import load_cases_from_json
 from evals.langfuse_reporter import LangfuseClient, LangfuseReporter
-from evals.utils import get_report_summary
 
 
 def main():
@@ -31,10 +30,10 @@ def main():
     
     # Display SDK results
     print("\n" + "="*60)
-    reports[0].display()
+    reports[0].run_display()
     
     # Get summary
-    summary = get_report_summary(reports[0])
+    summary = reports[0].get_summary()
     print(f"\n📊 Summary:")
     print(f"   Pass Rate: {summary['pass_rate']:.1%}")
     print(f"   Average Score: {summary['average_score']:.2f}")
@@ -44,12 +43,11 @@ def main():
     lf_client = LangfuseClient()
     reporter = LangfuseReporter(lf_client)
     case_names = [case.name for case in cases]
-    evaluator_names = [type(e).__name__ for e in experiment.evaluators]
-    trace_ids = reporter.log_experiment_results(reports, case_names, evaluator_names)
+    trace_ids = reporter.log_experiment_results(reports, case_names)
     print(f"   Logged {len(trace_ids)} traces to Langfuse")
     
     # Save experiment
-    experiment.to_file("experiment_files/trajectory_eval")
+    experiment.to_file("trajectory_eval", "json")
     print("\n💾 Experiment saved to ./experiment_files/trajectory_eval.json")
 
 

evals/utils.py

@@ -1,15 +0,0 @@
-"""
-Utility functions for evaluations.
-"""
-
-def get_report_summary(report):
-    """
-    Calculate summary metrics from an EvaluationReport.
-    
-    Returns:
-        dict: A dictionary containing 'pass_rate' and 'average_score'.
-    """
-    return {
-        "pass_rate": sum(report.test_passes) / len(report.test_passes) if report.test_passes else 0,
-        "average_score": report.overall_score
-    }

evaluation/dataset.json

@@ -230,85 +230,5 @@
       "Feature importance ranking",
       "Impact of each feature on the model's decision"
     ]
-  },
-  {
-    "id": "synth_sdk_1",
-    "question": "Application ID: 1023456",
-    "expected_intent": "General",
-    "expected_answer_key_points": [
-      "Fraud Score: 750\nRisk Level: High\nSHAP Values:\n- Transaction History Impact: +200\n- Credit Score Impact: +150\n- Recent Loan Applications Impact: -50\n- Employment History Impact: +100\nExplanation: The application has a high fraud score primarily due to the transaction history and credit score, which together contribute significantly to the high risk level. The recent loan applications positively influence the score by lowering it slightly, but not enough to offset the other risk factors. Employment history further increases the score. It's essential to focus on improving transaction patterns to lower the fraud score."
-    ]
-  },
-  {
-    "id": "synth_sdk_2",
-    "question": "Application ID: 4321XYZ\n- Request from Fraud Analyst: \"Please provide a detailed breakdown of the fraud score for this application. Include key feature contributions and provide an explanation for any features that heavily influence the score.\"\n- Tools and Capabilities Requested:\n  - use explain_fraud_score(4321XYZ) to highlight SHAP values.\n  - identify high-contributing features impacting fraud score.\n  - analyze borderline scores and explain if any apply.",
-    "expected_intent": "General",
-    "expected_answer_key_points": [
-      "Fraud Score: 680 (Moderate Risk)\nKey SHAP Contribution Analysis:\n1. \"Credit History Length\": +45 (Positive impact due to long credit history)\n2. \"Number of Recent Applications\": +100 (Negative impact due to recent application spike)\n3. \"Unusual Geolocation Activity\": +150 (High negative impact due to recent activity in a high-risk zone)\n4. \"Income Verification\": -65 (Positive impact due to verified and stable income)\n\nExplanation:\n- The fraud score is boosted to a higher risk mainly due to the unusual geolocation activity and the number of recent applications, which are significant red flags in fraud detection. While the credit history length and verified income provide a buffer reducing the risk level somewhat, the overall risk remains moderate.\n- Due to the combination and magnitude of these factors, the borderline score reflects an increased risk with possibilities of both legitimate and fraudulent activities present."
-    ]
-  },
-  {
-    "id": "synth_sdk_3",
-    "question": "Task: Analyze the performance of the fraud detection model over the past six months.\n\nData Provided: \n- List of application IDs from the last six months\n- Access to tools: compare_to_population, explain_fraud_score\n\nKey Question: What trends can be identified in the distribution of fraud scores and SHAP value importance over this period?",
-    "expected_intent": "General",
-    "expected_answer_key_points": [
-      "Output:\n1. Statistical report indicating any significant trends or shifts in fraud scores over the six-month period using compare_to_population.\n2. Identification of any changes in SHAP feature importance that may indicate shifts in model predictions.\n3. Summary of observations highlighting potential model drift or changes in population characteristics that could impact fraud detection accuracy."
-    ]
-  },
-  {
-    "id": "synth_sdk_4",
-    "question": "A data scientist is analyzing the fraud detection model's performance over the past six months for applications in a specific region. They want to understand how the model's scoring distribution has shifted over this period and identify any significant changes in feature importance that might correspond with this shift. Use the tools compare_to_population and explain_fraud_score. The focus is on applications in the Midwest region, with application IDs ranging from 1001 to 1050. The model should track score distribution shifts and highlight any significant changes in SHAP feature importance.",
-    "expected_intent": "General",
-    "expected_answer_key_points": [
-      "1. Provide a trend report showing score distribution changes over the past six months for the specified application range in the Midwest, indicating whether fraud scores have increased, decreased, or remained stable.\n2. Use compare_to_population to determine how these changes compare to national trends, noting any significant deviations.\n3. Highlight significant shifts in SHAP value contributions for key features over the same period, indicating which features have become more or less significant in model decision-making.\n4. Correlate findings with any historical applications data to suggest possible reasons for observed shifts, such as policy changes or external economic factors."
-    ]
-  },
-  {
-    "id": "synth_sdk_5",
-    "question": "The Compliance Officer wants to verify if there is any evidence of gender bias in fraud scoring for the application ID: F12345. Use the 'check_fair_lending_flags' to detect bias indicators and 'compare_to_population' to analyze demographic-specific fraud score patterns.",
-    "expected_intent": "General",
-    "expected_answer_key_points": [
-      "The 'check_fair_lending_flags' tool indicates a potential bias against female applicants due to disparate impact ratios below threshold in the fraud scoring process, indicating females are disproportionately denied compared to males. 'compare_to_population' further shows that female applicants have consistently higher average fraud scores compared to male counterparts for similar profiles, suggesting an underlying model bias."
-    ]
-  },
-  {
-    "id": "synth_sdk_6",
-    "question": "compliance_officer: \"I need to verify if application 1234 has any fair lending bias issues. Check for any bias flags and provide basic SHAP contributions for this case.\"",
-    "expected_intent": "General",
-    "expected_answer_key_points": [
-      "1. Use check_fair_lending_flags(app_id=1234) to identify if there are any bias issues. The tool will return 'No Bias Detected' or 'Potential Bias Detected'.\n2. If bias is present, explain_fraud_score(app_id=1234) to understand which features contribute most to the score.\n3. Report: 'No Bias Detected. Main factors influencing the fraud score are {feature1, feature2} contributing {value1, value2}.' OR 'Potential Bias Detected. SHAP analysis reveals major contributors: {biased_feature} with {value}.'"
-    ]
-  },
-  {
-    "id": "synth_sdk_7",
-    "question": "You are tasked with preparing an executive summary that highlights the current trends in fraud detection for the past quarter for the executive team of the financial services company. Use the following application IDs: ['A12345', 'B67890', 'C13579'].",
-    "expected_intent": "General",
-    "expected_answer_key_points": [
-      "The executive summary should include:\n1. A concise summary of the fraud scores and risk levels for each application using 'get_application_summary' for the IDs 'A12345', 'B67890', 'C13579'.\n2. A high-level analysis showing any evident trends in fraud scores over the quarter period.\n3. A brief mention of major contributing factors to the high fraud scores, utilizing SHAP values (not the specifics, just an overview of the top features contributing).\n4. A concise explanation of the application network status for each of these IDs using 'get_identity_network'.\n\nThe summary should be formatted in a report-friendly manner, prioritizing clarity and brevity for executive review."
-    ]
-  },
-  {
-    "id": "synth_sdk_8",
-    "question": "Generate a high-level summary for executives on the fraud score trends and key contributing factors for applications over the last quarter. Include how applications are linked within the identity network for potential collusion. Use the following application IDs: ['A123', 'B456', 'C789'].",
-    "expected_intent": "General",
-    "expected_answer_key_points": [
-      "The analysis for the last quarter indicates a rising trend in fraud scores, with the average score increasing by approximately 15% compared to the previous period. Key contributing factors identified through SHAP analysis include unusually high transaction volumes and discrepancies in reported income. Specific to applications ['A123', 'B456', 'C789'], notable features contributing to high fraud scores were similar across the board, indicating potential systemic fraud patterns. Moreover, the get_identity_network for these applications reveals interconnected relationships, suggesting possible collusion attempts. The network analysis indicates that several other applications are linked through shared contact information and transaction patterns, necessitating further investigation into these connections. Overall, the data suggests an evolving strategy that involves increasing sophistication in application profiles that require continued vigilance and adaptation of fraud detection measures."
-    ]
-  },
-  {
-    "id": "synth_sdk_9",
-    "question": "Application ID: 12345; Requested Action: Analyze impact of linked applications on fraud score using get_identity_network.",
-    "expected_intent": "General",
-    "expected_answer_key_points": [
-      "get_identity_network(12345) returns {\"linked_applications\": [\"11111\", \"22222\", \"33333\"]}. explain_fraud_score(12345) returns {\"SHAP_values\": {\"income\": -100, \"credit_history\": 50, \"linked_applications\": 200}}. compare_to_population(12345) shows linked applications influence fraud score to be 20% higher than similar approved applications. Risk level classified as 'Moderate Risk' due to negative impact from linked applications."
-    ]
-  },
-  {
-    "id": "synth_sdk_10",
-    "question": "Application ID: 12345\nTask: Investigate the fraud score and impact of linked applications using the get_identity_network tool. Provide an analysis of how these linked applications influence the fraud score and SHAP values for Application ID 12345. Use compare_to_population to understand how these impacts differ from the general population.",
-    "expected_intent": "General",
-    "expected_answer_key_points": [
-      "1. Identity Network Output: Application 12345 is linked with applications 56789, 98765, and 65432.\n2. Fraud Score Analysis: The fraud score for Application 12345 is 850. Linked applications have high fraud scores: Application 56789 (900), Application 98765 (875), and Application 65432 (910).\n3. SHAP Values: Key features that increased the fraud score include 'cross-application suspicious transactions' and 'irregular account activities.'\n4. Population Comparison: Compared to the approved applications' average fraud score of 500, Application 12345 and its linked applications show significantly higher risk, suggesting a potential fraud ring.\n5. Conclusion: The linkage to high-risk applications within the identity network explains the elevated fraud score for Application 12345, highlighting the influence of network associations in detecting fraud patterns."
-    ]
   }
 ]