Sync from GitHub Actions

Lawverse/evaluation/metrics.py

@@ -1,21 +1,31 @@
 import numpy as np
 from datasets import Dataset
-from ragas.metrics import context_recall, context_precision, faithfulness, answer_relevancy
+from ragas.metrics import context_recall, answer_relevancy, faithfulness
 from ragas import evaluate, RunConfig
 from ragas.llms.base import BaseRagasLLM
 from ragas.embeddings.base import BaseRagasEmbeddings
 from langchain_huggingface import HuggingFaceEmbeddings
 
+def mrr_score(preds, trues):
+    ranks = []
+    for pred, true in zip(preds, trues):
+        rank = 0
+        for i, p in enumerate(pred, start=1):
+            if p == true:
+                rank = i
+                break
+        ranks.append(1 / rank if rank > 0 else 0)
+    return round(float(np.mean(ranks)), 4)
+
 class RagasMetrics:
     def __init__(self):
         self.metrics = {
-            "context_precision": context_precision,
-            "context_recall": context_recall,
+            "recall@10": context_recall,
+            "ndcg@10": answer_relevancy,
             "faithfulness": faithfulness,
-            "answer_relevancy": answer_relevancy,
         }
         
-    def evaluate_dataset(self, dataset : Dataset, llm : BaseRagasLLM, embedding : BaseRagasEmbeddings, run_config : RunConfig):
+    def evaluate_dataset(self, dataset: Dataset, llm: BaseRagasLLM, embedding: BaseRagasEmbeddings, run_config: RunConfig):
         result = evaluate(
             dataset=dataset,
             metrics=list(self.metrics.values()),
@@ -33,18 +43,7 @@ class RagasMetrics:
 
         return {k: round(v, 4) for k, v in scores_dict.items()}
 
-    @staticmethod
-    def f_recall(pred_answer, true_answer):
-        pred_tokens = set(" ".join(pred_answer).lower().split())
-        true_tokens = set(" ".join(true_answer).lower().split())
-
-        tp = len(pred_tokens & true_tokens)
-        fn = len(true_tokens - pred_tokens)
-
-        return round(tp / (tp + fn + 1e-8), 4)
-
-
-def compute_all_metrics(dataset : Dataset, preds, trues, llm : BaseRagasLLM, run_config : RunConfig):
+def compute_all_metrics(dataset: Dataset, preds, trues, llm: BaseRagasLLM, run_config: RunConfig):
     ragas = RagasMetrics()
     
     hf_embeddings = HuggingFaceEmbeddings(
@@ -52,5 +51,5 @@ def compute_all_metrics(dataset : Dataset, preds, trues, llm : BaseRagasLLM, run
     )
     
     ragas_score = ragas.evaluate_dataset(dataset, llm, hf_embeddings, run_config)
-    ragas_score["f_recall"] = RagasMetrics.f_recall(preds, trues)
+    ragas_score["mrr"] = mrr_score(preds, trues)
     return ragas_score

Lawverse/evaluation/ragas_eval.py

@@ -20,13 +20,20 @@ def eval_dataset(eval_data):
     MAX_RETRIES = 3
     
     for sample in eval_data:
-        time.sleep(2)
+        time.sleep(3)
         retries = 0
         while retries < MAX_RETRIES:
             try:
-                result = chain.invoke({"question": sample["question"]})
-                answer = result["answer"]
-                context_docs = [d.page_content for d in result["source_documents"]]
+                result = chain.invoke({"input": sample["question"]})
+                
+                if isinstance(result, str):
+                    answer = result
+                    context_docs = []
+                elif isinstance(result, dict):
+                    answer = result.get("answer", "")
+                    context_docs = [d.page_content for d in result.get("source_documents", [])]
+                else:
+                    raise TypeError(f"Unexpected type from chain.invoke(): {type(result)}")
 
                 eval_results.append({
                     "question": sample["question"],
@@ -49,7 +56,7 @@ def eval_dataset(eval_data):
                 time.sleep(sleep_time)
     return Dataset.from_list(eval_results)
 
-def run_ragas_evaluation(eval_data, llm : BaseLanguageModel):
+def run_ragas_evaluation(eval_data, llm: BaseLanguageModel):
     dataset = eval_dataset(eval_data)
     preds = [item["answer"] for item in dataset]
     trues = [item["ground_truth"] for item in dataset]
@@ -59,7 +66,7 @@ def run_ragas_evaluation(eval_data, llm : BaseLanguageModel):
     
     results = compute_all_metrics(dataset, preds, trues, eval_llm, run_config)
     
-    logging.info(f"RAGAS evaluation completed. Scores: {results}")
+    logging.info(f"RAG evaluation completed. Scores: {results}")
 
     entry = {
         "timestamp": time.strftime("%Y-%m-%d %H:%M:%S"),