from typing import Optional, Union import weave from deepeval.metrics import SummarizationMetric from deepeval.test_case import LLMTestCase from weave import Scorer class SummarizationScorer(Scorer): """Summarization Scorer. This scorer uses DeepEval's `Summarization` Metric to assess how well the generated output aligns with the reference context. The summarization metric uses LLMs to determine whether the LLM application is generating factually correct summaries while including the neccessary details from the original text. Attributes: threshold (float): Minimum passing threshold, defaults to 0.5. model (str): LLM model for scoring, defaults to "gpt-4". assessment_questions: a list of close-ended questions that can be answered with either a 'yes' or a 'no'. These are questions you want your summary to be able to ideally answer, and is especially helpful if you already know what a good summary for your use case looks like. If include_reason (bool): Include reason for the evaluation score, defaults to True. strict_mode (bool): Enforces binary metric scoring (1 or 0), defaults to False. async_mode (bool): Use asynchronous scoring, defaults to True. verbose (bool): Print intermediate steps used for scoring, defaults to False. truths_extraction_limit (Optional[int]): Maximum number of factual truths to extract from the retrieval_context. Defaults to None. metric (SummarizationMetric): An instance of DeepEval's `SummarizationMetric` for scoring. """ threshold: float = Optional[None] model: str = Optional[None] include_reason: bool = Optional[None] strict_mode: bool = Optional[None] async_mode: bool = Optional[None] verbose: bool = Optional[None] assessment_questions: Optional[list[str]] = Optional[None] n: Optional[int] = Optional[None] truths_extraction_limit: Optional[int] = Optional[None] metric: SummarizationMetric = Optional[None] def __init__( self, threshold: float = 0.5, model: str = "gpt-4", include_reason: bool = True, strict_mode: bool = False, async_mode: bool = True, verbose: bool = False, assessment_questions: Optional[list[str]] = None, n: Optional[int] = 5, truths_extraction_limit: Optional[int] = None, ): """Initialize the Summarization Scorer with DeepEval's Summarization Metric. Args: threshold (float): Minimum passing threshold, defaults to 0.5. model (str): LLM model for scoring, defaults to "gpt-4". include_reason (bool): Include reason for the evaluation score, defaults to True. strict_mode (bool): Enforces binary metric scoring (1 or 0), defaults to False. async_mode (bool): Use asynchronous scoring, defaults to True. verbose (bool): Print intermediate steps used for scoring, defaults to False. assessment_questions (Optional[list[str]]): a list of close-ended questions that can be answered with either a 'yes' or a 'no'. These are questions you want your summary to be able to ideally answer, and is especially helpful if you already know what a good summary for your use case looks like. If `assessment_questions` is not provided, the metric will generate a set of `assessment_questions` at evaluation time. n (Optional[int]): The number of assessment questions to generate when `assessment_questions` is not provided. Defaults to 5. truths_extraction_limit (Optional[int]): Maximum number of factual truths to extract from the retrieval_context. Defaults to None. """ super().__init__( threshold=threshold, model=model, include_reason=include_reason, strict_mode=strict_mode, async_mode=async_mode, verbose=verbose, assessment_questions=assessment_questions, n=n, truths_extraction_limit=truths_extraction_limit, ) self.threshold = threshold self.model = model self.include_reason = include_reason self.strict_mode = strict_mode self.async_mode = async_mode self.verbose = verbose self.assessment_questions = assessment_questions self.n = n self.truths_extraction_limit = truths_extraction_limit self.metric = SummarizationMetric( threshold=self.threshold, model=self.model, include_reason=self.include_reason, async_mode=self.async_mode, strict_mode=self.strict_mode, verbose_mode=self.verbose, assessment_questions=self.assessment_questions, n=self.n, truths_extraction_limit=self.truths_extraction_limit, ) @weave.op def score( self, input: str, actual_output: str, expected_output: Optional[str] = None, retrieval_context: Optional[list[str]] = None, context: Optional[list[str]] = None, ) -> dict[str, Union[str, float]]: """Evaluate the quality of summarization of an LLM generated response. The Summarization score is calculated according to the following equation: Summarization = min(Alignment Score, Coverage Score) where, - Alignment Score: determines whether the summary contains hallucinated or contradictory information to the original text. - Coverage Score: determines whether the summary contains the neccessary information from the original text. While the Alignment Score is similar to that of the Hallucination Score, the Coverage Score is first calculated by generating n closed-ended questions that can only be answered with either a 'yes or a 'no', before calculating the ratio of which the original text and summary yields the same answer. Args: input (str): The input query or prompt that triggered the output. actual_output (str): The LLM generated response to evaluate. expected_output (Optional[str]): The expected or reference output, defaults to None. retrieval_context (Optional[list[str]]): The context containing factual information to compare against. context (Optional[list[str]]): Additional context for the evaluation, defaults to None. Returns: dict[str, Union[str, float]]: A dictionary containing: - "score" (float): The computed summarization score. - "reason" (str): A detailed explanation for the assigned score. """ test_case = LLMTestCase( input=input, actual_output=actual_output, expected_output=expected_output, retrieval_context=retrieval_context, context=context, ) result: dict[str, Union[str, float]] = {} self.metric.measure(test_case) result = {"score": self.metric.score, "reason": self.metric.reason} return result