evals/run_helpfulness.py

#!/usr/bin/env python
"""
Run helpfulness evaluation on the fraud agent.

This is equivalent to basic_eval.py from the SDK quickstart,
but customized for the fraud explainability use case.
"""
from strands_evals import Experiment
from evals.config import helpfulness_evaluator
from evals.task_functions import get_fraud_explanation
from evals.dataset_loader import load_cases_from_json
from evals.langfuse_reporter import LangfuseClient, LangfuseReporter
from evals.utils import get_report_summary


def main():
    print("=== Helpfulness Evaluation ===\n")
    
    # Load test cases
    cases = load_cases_from_json("evaluation/dataset.json")
    print(f"Loaded {len(cases)} test cases\n")
    
    # Create experiment
    experiment = Experiment(
        cases=cases,
        evaluators=[helpfulness_evaluator]
    )
    
    # Run evaluations
    print("Running evaluations...")
    reports = experiment.run_evaluations(get_fraud_explanation)
    
    # Display SDK results
    print("\n" + "="*60)
    reports[0].display()
    
    # Get summary
    summary = get_report_summary(reports[0])
    print(f"\n📊 Summary:")
    print(f"   Pass Rate: {summary['pass_rate']:.1%}")
    print(f"   Average Score: {summary['average_score']:.2f}")
    
    # Log to Langfuse
    print("\n📤 Logging to Langfuse...")
    lf_client = LangfuseClient()
    reporter = LangfuseReporter(lf_client)
    case_names = [case.name for case in cases]
    evaluator_names = [type(e).__name__ for e in experiment.evaluators]
    trace_ids = reporter.log_experiment_results(reports, case_names, evaluator_names)
    print(f"   Logged {len(trace_ids)} traces to Langfuse")
    
    # Save experiment
    experiment.to_file("experiment_files/helpfulness_eval")
    print("\n💾 Experiment saved to ./experiment_files/helpfulness_eval.json")


if __name__ == "__main__":
    main()