Update evaluator_module.py

evaluator_module.py

@@ -5,6 +5,7 @@ import json
 import torch
 from collections import defaultdict
 import spacy
+import evaluate
 from transformers import pipeline
 from sentence_transformers import SentenceTransformer
 from transformers import AutoTokenizer, AutoModelForSequenceClassification
@@ -35,6 +36,8 @@ class AetherScoreEvaluator:
         self.sentence_model = SentenceTransformer('all-MiniLM-L6-v2')
 
         # for hallucination
+        self.rouge = evaluate.load("rouge")
+        self.sacrebleu = evaluate.load("sacrebleu")
         self.nli_tokenizer = AutoTokenizer.from_pretrained("prajjwal1/bert-mini-mnli")
         self.nli_model = AutoModelForSequenceClassification.from_pretrained("prajjwal1/bert-mini-mnli")
 
@@ -47,39 +50,55 @@ class AetherScoreEvaluator:
 
     def _evaluate_with_llm_judge(self, prompt: str, response: str) -> dict:
         """
-        Hallucination detection using Embedding similarity + NLI.
-        Returns scores + detailed explanations for both hallucination and assumption.
+        Hallucination detection using:
+        - NLI (entailment, neutral, contradiction)
+        - Embedding similarity
+        - ROUGE-L
+        - SacreBLEU
+        Assumption control derived from NLI neutrality.
         """
-        # Step 1: Embedding similarity (prompt → response)
+        # Step 1: Embedding similarity
         emb_sim = self._semantic_similarity(prompt, response)
-
-        # Step 2: NLI check (entailment, neutral, contradiction)
-        inputs = self.nli_tokenizer.encode_plus(
-            prompt, response, return_tensors="pt", truncation=True
-        )
+    
+        # Step 2: NLI inference
+        inputs = self.nli_tokenizer.encode_plus(prompt, response, return_tensors="pt", truncation=True)
         with torch.no_grad():
             logits = self.nli_model(**inputs).logits
             probs = torch.softmax(logits, dim=-1).cpu().numpy()[0]
             entailment, neutral, contradiction = probs[2], probs[1], probs[0]
-
-        # Step 3: Compute hallucination & assumption
-        halluc_score = 1 - ((entailment * 0.7) + (emb_sim * 0.3))
-        assumption_score = 1 - halluc_score  # alternative proxy
-
-        # Step 4: Explanations
+    
+        # Step 3: ROUGE-L
+        rouge_l = self.rouge.compute(predictions=[response], references=[prompt])["rougeL"]
+    
+        # Step 4: SacreBLEU (normalized 0–1)
+        sacrebleu = self.sacrebleu.compute(predictions=[response], references=[[prompt]])["score"] / 100.0
+    
+        # Step 5: Weighted hallucination score
+        weights = {"entailment": 0.4, "embedding": 0.2, "rouge": 0.2, "sacrebleu": 0.2}
+        halluc_score = 1 - (
+            weights["entailment"] * entailment +
+            weights["embedding"] * emb_sim +
+            weights["rouge"] * rouge_l +
+            weights["sacrebleu"] * sacrebleu
+        )
+    
+        # Step 6: Assumption control from neutrality
+        assumption_score = 1 - neutral
+    
+        # Step 7: Explanations
         halluc_expl = (
-            f"Embedding similarity={emb_sim:.2f}, "
-            f"NLI entailment={entailment:.2f}, neutral={neutral:.2f}, contradiction={contradiction:.2f}"
+            f"Entailment={entailment:.2f}, Embedding={emb_sim:.2f}, "
+            f"ROUGE-L={rouge_l:.2f}, SacreBLEU={sacrebleu:.2f}, Neutral={neutral:.2f}"
         )
         assumption_expl = (
-            f"Assumption control leverages NLI neutrality={neutral:.2f}. "
-            "Higher neutrality indicates more uncertainty; lower means fewer assumptions."
+            f"Assumption control is derived from NLI neutrality={neutral:.2f}. "
+            "Lower neutrality → stronger confidence."
         )
-
+    
         return {
             "hallucination_score": (float(halluc_score), halluc_expl),
             "assumption_control": (float(assumption_score), assumption_expl),
-        }
+    }
 
     # Single Evaluation # Inputs-->> Prompt, Agent Response, Expected Answer(Optional), Agent Name and Task type( General, QA, Summarizaton)etc 
     def evaluate_single(self, prompt: str, response: str, expected_answer: Optional[str] = None, task_type: str = "general") -> Dict: