Update app.py

app.py

@@ -361,14 +361,8 @@ import mauve
 from sacrebleu import corpus_bleu
 from rouge_score import rouge_scorer
 from bert_score import score
-from transformers import GPT2LMHeadModel, GPT2Tokenizer, pipeline
-import nltk
-from nltk.util import ngrams
-from nltk.tokenize import word_tokenize
-from nltk.translate.meteor_score import meteor_score
-from nltk.translate.chrf_score import sentence_chrf
-from textstat import flesch_reading_ease, flesch_kincaid_grade
-from sklearn.metrics.pairwise import cosine_similarity
+from transformers import GPT2LMHeadModel, GPT2Tokenizer, pipeline, AutoTokenizer
+import re
 from mauve import compute_mauve
 import os
 import gradio as gr
@@ -406,11 +400,9 @@ class RAGEvaluator:
     def __init__(self):
         self.gpt2_model, self.gpt2_tokenizer = self.load_gpt2_model()
         self.bias_pipeline = pipeline("zero-shot-classification", model="Hate-speech-CNERG/dehatebert-mono-english")
-        # Download required NLTK data
-        nltk.download('punkt', quiet=True)
-        nltk.download('wordnet', quiet=True)
-        nltk.download('omw-1.4', quiet=True)
-
+        # Initialize tokenizer for text processing
+        self.tokenizer = AutoTokenizer.from_pretrained("bert-base-uncased")
+        
     def load_gpt2_model(self):
         model = GPT2LMHeadModel.from_pretrained('gpt2')
         tokenizer = GPT2Tokenizer.from_pretrained('gpt2')
@@ -449,8 +441,17 @@ class RAGEvaluator:
         return ppl.item()
 
     def evaluate_diversity(self, texts):
-        all_tokens = [tok for text in texts for tok in text.split()]
-        unique_bigrams = set(ngrams(all_tokens, 2))
+        # Use Hugging Face tokenizer instead of NLTK
+        all_tokens = []
+        for text in texts:
+            tokens = self.tokenizer.tokenize(text)
+            all_tokens.extend(tokens)
+        
+        # Create bigrams manually
+        unique_bigrams = set()
+        for i in range(len(all_tokens) - 1):
+            unique_bigrams.add((all_tokens[i], all_tokens[i+1]))
+        
         diversity_score = len(unique_bigrams) / len(all_tokens) if all_tokens else 0
         return diversity_score
 
@@ -460,19 +461,79 @@ class RAGEvaluator:
         return bias_score
 
     def evaluate_meteor(self, candidates, references):
-        meteor_scores = [
-            meteor_score([word_tokenize(ref)], word_tokenize(cand))
-            for ref, cand in zip(references, candidates)
-        ]
-        return sum(meteor_scores) / len(meteor_scores)
+        # Simple approximation of METEOR without NLTK
+        # This is a simplified version - consider using an external API for full METEOR
+        meteor_scores = []
+        for ref, cand in zip(references, candidates):
+            ref_tokens = self.tokenizer.tokenize(ref)
+            cand_tokens = self.tokenizer.tokenize(cand)
+            
+            # Calculate precision and recall
+            common_tokens = set(ref_tokens) & set(cand_tokens)
+            precision = len(common_tokens) / len(cand_tokens) if cand_tokens else 0
+            recall = len(common_tokens) / len(ref_tokens) if ref_tokens else 0
+            
+            # F-measure with alpha=0.9 (METEOR default)
+            if precision + recall == 0:
+                f_score = 0
+            else:
+                f_score = (10 * precision * recall) / (9 * precision + recall)
+            
+            meteor_scores.append(f_score)
+        
+        return sum(meteor_scores) / len(meteor_scores) if meteor_scores else 0
     
     def evaluate_chrf(self, candidates, references):
-        chrf_scores = [sentence_chrf(ref, cand) for ref, cand in zip(references, candidates)]
-        return sum(chrf_scores) / len(chrf_scores)
+        # Simple character n-gram F-score approximation
+        chrf_scores = []
+        for ref, cand in zip(references, candidates):
+            # Character 6-grams
+            ref_chars = list(ref)
+            cand_chars = list(cand)
+            
+            ref_ngrams = set()
+            cand_ngrams = set()
+            
+            # Create character 6-grams
+            for i in range(len(ref_chars) - 5):
+                ref_ngrams.add(tuple(ref_chars[i:i+6]))
+            
+            for i in range(len(cand_chars) - 5):
+                cand_ngrams.add(tuple(cand_chars[i:i+6]))
+            
+            common_ngrams = ref_ngrams & cand_ngrams
+            precision = len(common_ngrams) / len(cand_ngrams) if cand_ngrams else 0
+            recall = len(common_ngrams) / len(ref_ngrams) if ref_ngrams else 0
+            
+            if precision + recall == 0:
+                chrf_score = 0
+            else:
+                chrf_score = 2 * precision * recall / (precision + recall)
+            
+            chrf_scores.append(chrf_score)
+        
+        return sum(chrf_scores) / len(chrf_scores) if chrf_scores else 0
     
     def evaluate_readability(self, text):
-        flesch_ease = flesch_reading_ease(text)
-        flesch_grade = flesch_kincaid_grade(text)
+        # Simple readability metrics without textstat
+        words = re.findall(r'\b\w+\b', text.lower())
+        sentences = re.split(r'[.!?]+', text)
+        
+        num_words = len(words)
+        num_sentences = len([s for s in sentences if s.strip()])
+        
+        # Average word length
+        avg_word_length = sum(len(word) for word in words) / num_words if num_words else 0
+        
+        # Words per sentence
+        words_per_sentence = num_words / num_sentences if num_sentences else 0
+        
+        # Simplified Flesch Reading Ease approximation
+        flesch_ease = 206.835 - (1.015 * words_per_sentence) - (84.6 * avg_word_length)
+        
+        # Simplified Flesch-Kincaid Grade Level approximation
+        flesch_grade = (0.39 * words_per_sentence) + (11.8 * avg_word_length) - 15.59
+        
         return flesch_ease, flesch_grade
         
     def evaluate_mauve(self, reference_texts, generated_texts):