update accuracy code

app.py

@@ -22,6 +22,19 @@ import tqdm
 import accelerate
 import re
 
+
+import torch
+from sacrebleu import corpus_bleu
+from rouge_score import rouge_scorer
+from bert_score import score
+from transformers import GPT2LMHeadModel, GPT2Tokenizer, pipeline
+import nltk
+from nltk.util import ngrams
+
+
+
+
+
 api_key = os.getenv('API_KEY')
 
 
@@ -249,30 +262,136 @@ def format_chat_history(message, chat_history):
     return formatted_chat_history
     
 
-def conversation(qa_chain, message, history):
+
+
+
+###############################################
+class RAGEvaluator:
+    def __init__(self):
+        self.gpt2_model, self.gpt2_tokenizer = self.load_gpt2_model()
+        self.bias_pipeline = pipeline("zero-shot-classification", model="Hate-speech-CNERG/dehatebert-mono-english")
+
+    def load_gpt2_model(self):
+        model = GPT2LMHeadModel.from_pretrained('gpt2')
+        tokenizer = GPT2Tokenizer.from_pretrained('gpt2')
+        return model, tokenizer
+
+    def evaluate_bleu_rouge(self, candidates, references):
+        bleu_score = corpus_bleu(candidates, [references]).score
+        scorer = rouge_scorer.RougeScorer(['rouge1', 'rouge2', 'rougeL'], use_stemmer=True)
+        rouge_scores = [scorer.score(ref, cand) for ref, cand in zip(references, candidates)]
+        rouge1 = sum([score['rouge1'].fmeasure for score in rouge_scores]) / len(rouge_scores)
+        return bleu_score, rouge1
+
+    def evaluate_bert_score(self, candidates, references):
+        P, R, F1 = score(candidates, references, lang="en", model_type='bert-base-multilingual-cased')
+        return P.mean().item(), R.mean().item(), F1.mean().item()
+
+    def evaluate_perplexity(self, text):
+        encodings = self.gpt2_tokenizer(text, return_tensors='pt')
+        max_length = self.gpt2_model.config.n_positions
+        stride = 512
+        lls = []
+        for i in range(0, encodings.input_ids.size(1), stride):
+            begin_loc = max(i + stride - max_length, 0)
+            end_loc = min(i + stride, encodings.input_ids.size(1))
+            trg_len = end_loc - i
+            input_ids = encodings.input_ids[:, begin_loc:end_loc]
+            target_ids = input_ids.clone()
+            target_ids[:, :-trg_len] = -100
+            with torch.no_grad():
+                outputs = self.gpt2_model(input_ids, labels=target_ids)
+                log_likelihood = outputs[0] * trg_len
+            lls.append(log_likelihood)
+        ppl = torch.exp(torch.stack(lls).sum() / end_loc)
+        return ppl.item()
+
+    def evaluate_diversity(self, texts):
+        all_tokens = [tok for text in texts for tok in text.split()]
+        unique_bigrams = set(ngrams(all_tokens, 2))
+        diversity_score = len(unique_bigrams) / len(all_tokens) if all_tokens else 0
+        return diversity_score
+
+    def evaluate_racial_bias(self, text):
+        results = self.bias_pipeline([text], candidate_labels=["hate speech", "not hate speech"])
+        bias_score = results[0]['scores'][results[0]['labels'].index('hate speech')]
+        return bias_score
+
+    def evaluate_all(self, question, response, reference):
+        candidates = [response]
+        references = [reference]
+        bleu, rouge1 = self.evaluate_bleu_rouge(candidates, references)
+        bert_p, bert_r, bert_f1 = self.evaluate_bert_score(candidates, references)
+        perplexity = self.evaluate_perplexity(response)
+        diversity = self.evaluate_diversity(candidates)
+        racial_bias = self.evaluate_racial_bias(response)
+        return {
+            "BLEU": bleu,
+            "ROUGE-1": rouge1,
+            "BERT P": bert_p,
+            "BERT R": bert_r,
+            "BERT F1": bert_f1,
+            "Perplexity": perplexity,
+            "Diversity": diversity,
+            "Racial Bias": racial_bias
+        }
+
+###################################
+
+evaluator = RAGEvaluator()
+
+
+#################################
+
+def display_metrics(metrics):
+    result = ""
+    for k, v in metrics.items():
+        if k == 'BLEU':
+            result += f"BLEU measures the overlap between the generated output and reference text based on n-grams. Higher scores indicate better match. Score obtained: {v}\n\n"
+        elif k == "ROUGE-1":
+            result += f"ROUGE-1 measures the overlap of unigrams between the generated output and reference text. Higher scores indicate better match. Score obtained: {v}\n\n"
+        elif k == 'BERT P':
+            result += "BERTScore evaluates the semantic similarity between the generated output and reference text using BERT embeddings.\n\n"
+            result += f"**BERT Precision**: {metrics['BERT P']}\n"
+            result += f"**BERT Recall**: {metrics['BERT R']}\n"
+            result += f"**BERT F1 Score**: {metrics['BERT F1']}\n\n"
+        elif k == 'Perplexity':
+            result += f"Perplexity measures how well a language model predicts the text. Lower values indicate better fluency and coherence. Score obtained: {v}\n\n"
+        elif k == 'Diversity':
+            result += f"Diversity measures the uniqueness of bigrams in the generated output. Higher values indicate more diverse and varied output. Score obtained: {v}\n\n"
+        elif k == 'Racial Bias':
+            result += f"Racial Bias score indicates the presence of biased language in the generated output. Higher scores indicate more bias. Score obtained: {v}\n\n"
+    return result
+
+def conversation(qa_chain, message, history, evaluator):
     formatted_chat_history = format_chat_history(message, history)
-    #print("formatted_chat_history",formatted_chat_history)
-   
-    # Generate response using QA chain
+    question_by_user = message
+    
     response = qa_chain({"question": message, "chat_history": formatted_chat_history})
     response_answer = response["answer"]
+    answer_of_question = response["answer"]
     if response_answer.find("Helpful Answer:") != -1:
         response_answer = response_answer.split("Helpful Answer:")[-1]
     response_sources = response["source_documents"]
+    context = " ".join([d.page_content for d in response_sources])
+    
     response_source1 = response_sources[0].page_content.strip()
     response_source2 = response_sources[1].page_content.strip()
     response_source3 = response_sources[2].page_content.strip()
-    # Langchain sources are zero-based
+
     response_source1_page = response_sources[0].metadata["page"] + 1
     response_source2_page = response_sources[1].metadata["page"] + 1
     response_source3_page = response_sources[2].metadata["page"] + 1
-    # print ('chat response: ', response_answer)
-    # print('DB source', response_sources)
-    
-    # Append user message and response to chat history
+
     new_history = history + [(message, response_answer)]
-    # return gr.update(value=""), new_history, response_sources[0], response_sources[1] 
-    return qa_chain, gr.update(value=""), new_history, response_source1, response_source1_page, response_source2, response_source2_page, response_source3, response_source3_page
+    
+    # Evaluate the metrics
+    metrics = evaluator.evaluate_all(question_by_user, answer_of_question, context)
+    evaluation_metrics = display_metrics(metrics)
+
+    return (qa_chain, gr.update(value=""), new_history, response_source1, response_source1_page,
+            response_source2, response_source2_page, response_source3, response_source3_page,
+            question_by_user, context, answer_of_question, evaluation_metrics)
     
 
 def upload_file(file_obj):
@@ -285,6 +404,30 @@ def upload_file(file_obj):
     return list_file_path
 
 
+# Function to display metrics
+def display_metrics(metrics):
+    result = ""
+    for k, v in metrics.items():
+        if k == 'BLEU':
+            result += f"BLEU measures the overlap between the generated output and reference text based on n-grams. Higher scores indicate better match. Score obtained: {v}\n\n"
+        elif k == "ROUGE-1":
+            result += f"ROUGE-1 measures the overlap of unigrams between the generated output and reference text. Higher scores indicate better match. Score obtained: {v}\n\n"
+        elif k == 'BERT P':
+            result += "BERTScore evaluates the semantic similarity between the generated output and reference text using BERT embeddings.\n\n"
+            result += f"**BERT Precision**: {metrics['BERT P']}\n"
+            result += f"**BERT Recall**: {metrics['BERT R']}\n"
+            result += f"**BERT F1 Score**: {metrics['BERT F1']}\n\n"
+        elif k == 'Perplexity':
+            result += f"Perplexity measures how well a language model predicts the text. Lower values indicate better fluency and coherence. Score obtained: {v}\n\n"
+        elif k == 'Diversity':
+            result += f"Diversity measures the uniqueness of bigrams in the generated output. Higher values indicate more diverse and varied output. Score obtained: {v}\n\n"
+        elif k == 'Racial Bias':
+            result += f"Racial Bias score indicates the presence of biased language in the generated output. Higher scores indicate more bias. Score obtained: {v}\n\n"
+    return result
+
+
+
+###################################
 def demo():
     with gr.Blocks(theme="base") as demo:
         vector_db = gr.State()
@@ -352,6 +495,21 @@ def demo():
             with gr.Row():
                 submit_btn = gr.Button("Submit message")
                 clear_btn = gr.ClearButton([msg, chatbot], value="Clear conversation")
+            with gr.Tab("Metrics"):
+                metrics_output = gr.Textbox(lines=10, label="Evaluation Metrics")
+        
+
+
+        with gr.Tab("Metrics"):
+            metrics_output = gr.Textbox(lines=10, label="Evaluation Metrics")
+        
+       
+
+
+  
+            
+    
+
             
         # Preprocessing events
         #upload_btn.upload(upload_file, inputs=[upload_btn], outputs=[document])
@@ -366,10 +524,13 @@ def demo():
             queue=False)
 
         # Chatbot events
-        msg.submit(conversation, \
-            inputs=[qa_chain, msg, chatbot], \
-            outputs=[qa_chain, msg, chatbot, doc_source1, source1_page, doc_source2, source2_page, doc_source3, source3_page], \
-            queue=False)
+        msg.submit(interact, inputs=[gr.State(), msg, history], outputs=[
+        gr.State(), chatbot, history, response_source1, response_source1_page,
+        response_source2, response_source2_page, response_source3, response_source3_page,
+        None, None, None, metrics_output
+    ])
+
+
         submit_btn.click(conversation, \
             inputs=[qa_chain, msg, chatbot], \
             outputs=[qa_chain, msg, chatbot, doc_source1, source1_page, doc_source2, source2_page, doc_source3, source3_page], \
@@ -378,6 +539,11 @@ def demo():
             inputs=None, \
             outputs=[chatbot, doc_source1, source1_page, doc_source2, source2_page, doc_source3, source3_page], \
             queue=False)
+
+
+        
+
+
     demo.queue().launch(debug=True)