evals/config.py

"""
Shared configuration for evaluators and models.
"""
from strands.models.openai import OpenAIModel
from strands_evals.evaluators import OutputEvaluator, TrajectoryEvaluator, FaithfulnessEvaluator

# Configure OpenAI model for evaluators
eval_model = OpenAIModel(model_id="gpt-4o")

# Helpfulness Evaluator
helpfulness_evaluator = OutputEvaluator(
    rubric="""
    Evaluate the response based on:
    1. Accuracy - Is the information factually correct?
    2. Completeness - Does it fully answer the question?
    3. Clarity - Is it easy to understand?
    
    Score 1.0 if all criteria are met excellently.
    Score 0.5 if some criteria are partially met.
    Score 0.0 if the response is inadequate or incorrect.
    """,
    include_inputs=True,
    model=eval_model
)

# Faithfulness Evaluator (Generic for now, to enable data generation)
faithfulness_evaluator = OutputEvaluator(
    rubric="""
    Evaluate if the response is faithful to the retrieved context.
    
    Score 1.0 if fully supported by context.
    Score 0.5 if partially supported or context unavailable.
    Score 0.0 if contains hallucinations or contradicts context.
    
    Penalize heavily for information NOT in the context.
    """,
    include_inputs=True,
    model=eval_model
)

# Trajectory Evaluator
trajectory_evaluator = TrajectoryEvaluator(
    rubric="""
    Evaluate the tool usage trajectory:
    1. Correct tool selection - Were the right tools chosen?
    2. Proper sequence - Logical order (Retrieve -> Analyze -> Explain)?
    3. Efficiency - No unnecessary tools?
    
    Score 1.0 if optimal tools used correctly.
    Score 0.5 if correct tools but suboptimal sequence.
    SCORE 0.0 if wrong tools or major inefficiencies.
    """,
    include_inputs=True,
    model=eval_model
)

# Key Points Evaluator (Custom)
from evals.key_points_evaluator import KeyPointsEvaluator
key_points_evaluator = KeyPointsEvaluator()