Update metrics_v2.py

metrics_v2.py

@@ -239,4 +239,66 @@ if __name__ == "__main__":
 
     print("\n----- Evaluation Result -----")
     print(f"Review Flag: {evaluation_result['review_flag']}")
-    print(f"Explanation: {evaluation_result['explanation']}")
+    print(f"Explanation: {evaluation_result['explanation']}")
+
+
+#######
+from typing import List, Tuple, Callable
+
+def evaluate_retrieval_precision(
+    questions: List[str], 
+    system: Callable[[str], List[str]], 
+    evaluator: Callable[[str, str], int], 
+    num_chunks_expected: int = 3,
+    verbose: bool = True
+) -> dict:
+  """
+  Evaluates the retrieval precision of a system using an LLM evaluator.
+
+  Args:
+    questions: A list of evaluation questions.
+    system: A function that takes a question as input and returns a list of retrieved chunks.
+    evaluator: A function that takes a question and a chunk as input and returns a relevance score (0 or 1).
+    num_chunks_expected: The number of chunks the system is expected to return. Defaults to 3.
+    verbose: Whether to print warnings for questions with fewer returned chunks than expected.
+
+  Returns:
+     A dictionary containing:
+      - 'mean_precision': The mean retrieval precision score across all questions.
+      - 'precision_scores': A list of precision scores for each individual question.
+      - 'question_relevance': A list of tuples, where each tuple contains a question and the number of relevant chunks retrieved for that question.
+
+  """
+
+  results = {
+      'mean_precision': 0.0,
+      'precision_scores': [],
+      'question_relevance': [] 
+  }
+
+  for i, question in enumerate(questions):
+    retrieved_chunks = system(question)
+
+    # Warning if fewer chunks are returned than expected
+    if len(retrieved_chunks) < num_chunks_expected and verbose:
+      print(f"Warning: System returned {len(retrieved_chunks)} chunks (expected {num_chunks_expected}) for question {i+1}: {question}")
+
+    # Calculate precision for the current question
+    relevant_chunks = sum(evaluator(question, chunk) for chunk in retrieved_chunks)
+    precision = relevant_chunks / len(retrieved_chunks) if retrieved_chunks else 0  
+    results['precision_scores'].append(precision)
+
+    # Store the question and its relevant chunk count
+    results['question_relevance'].append((question, relevant_chunks))
+
+  # Calculate mean precision 
+  results['mean_precision'] = sum(results['precision_scores']) / len(questions) if questions else 0
+  return results
+
+# Example usage (assuming you've defined 'questions', 'system', and 'evaluator'):
+evaluation_results = evaluate_retrieval_precision(
+    questions, system, evaluator, num_chunks_expected=3, verbose=True
+)
+print(f"Mean Retrieval Precision: {evaluation_results['mean_precision']:.2f}")
+print(f"Precision Scores for Each Question: {evaluation_results['precision_scores']}")
+print(f"Question Relevance: {evaluation_results['question_relevance']}")