Upload eval.py

eval.py

@@ -0,0 +1,198 @@
+import os
+from dotenv import load_dotenv
+import openai
+from gradio_client import Client
+
+# Load API Key from .env file
+load_dotenv()
+api_key = os.getenv("OPENAI_API_KEY")
+print(f"Using OpenAI API Key: {api_key[:5]}****{api_key[-3:]}")  
+openai.api_key = api_key
+
+# ---- STEP 1: Load First Aid Contextual Data ----
+from langchain_community.document_loaders import ArxivLoader
+from langchain.text_splitter import RecursiveCharacterTextSplitter
+from langchain_openai import OpenAIEmbeddings
+from langchain_community.vectorstores import Chroma
+from langchain.prompts import ChatPromptTemplate
+from langchain_openai import ChatOpenAI
+import wandb
+import pandas as pd
+
+# Load medical and first aid papers from ArXiv
+first_aid_docs = ArxivLoader(query="first aid treatment", load_max_docs=5).load()
+
+# Split documents for indexing
+text_splitter = RecursiveCharacterTextSplitter(chunk_size=250)
+docs = text_splitter.split_documents(first_aid_docs)
+
+# Create vectorstore
+vectorstore = Chroma.from_documents(docs, OpenAIEmbeddings())
+retriever = vectorstore.as_retriever(search_kwargs={"k": 2})
+
+# ---- Define First Aid Questions ----
+questions = [
+    "What are the first aid measures for high fever in infants?",
+    "What are the signs and symptoms of low blood sugar?",
+    "What does RICE stand for in first aid treatment?",
+    "What is the first aid treatment of bleeding?",
+    "What is the first aid management of burns?",
+    "What are the signs and symptoms of stroke?",
+    "What is the treatment of snake bite?",
+    "How do you provide first aid for choking?",
+    "What are the immediate steps to treat a fainting patient?",
+    "What are the First aid measures for taking care of a patient with insect stings and animal bites?"
+]
+
+# ---- STEP 2: Generate Ground Truth Responses using ChatGPT ----
+llm = ChatOpenAI(model_name="gpt-4", temperature=0)
+prompt_template = """
+Generate a detailed and accurate first-aid response based on the given context.
+
+### CONTEXT
+{context}
+
+### QUESTION
+{question}
+
+### RESPONSE
+"""
+prompt = ChatPromptTemplate.from_template(prompt_template)
+
+ground_truth_responses = []
+for question in questions:
+    retrieved_docs = retriever.invoke(question)
+    context_text = "\n".join([doc.page_content for doc in retrieved_docs])
+    generated_response = llm.invoke(prompt.format(context=context_text, question=question))
+    ground_truth_responses.append(str(generated_response))
+
+# ---- STEP 3: Fetch Responses from Deployed Chatbot ----
+print("\n===== Fetching Responses from Chatbot =====")
+
+client = Client("DrSyedFaizan/First_Aid_Assistant")
+responses = []
+
+for question in questions:
+    try:
+        result = client.predict(chatbot=[], message=question, api_name="/respond")
+        chat_history = result[1]  
+        chatbot_response = next((entry["content"] for entry in chat_history if entry["role"] == "assistant"), "[NO RESPONSE]")
+    except Exception as e:
+        chatbot_response = f"[ERROR: {e}]"
+    
+    responses.append(str(chatbot_response))
+
+# Save bot responses to a text file
+with open("bot_responses.txt", "w", encoding="utf-8") as f:
+    for q, r in zip(questions, responses):
+        f.write(f"Q: {q}\nA: {r}\n\n")
+
+# Print chatbot responses for debugging
+for q, r in zip(questions, responses):
+    print(f"Q: {q}\nA: {r}\n")
+
+# ---- STEP 5: Evaluate Using RAGAS ----
+from datasets import Dataset
+import pandas as pd
+from tqdm import tqdm
+from ragas import evaluate
+from ragas.metrics import (
+    answer_relevancy,
+    faithfulness,
+    context_recall,
+    answer_correctness,
+    answer_similarity
+)
+
+def create_ragas_dataset(eval_dataset):
+    """Convert dataset to RAGAS format."""
+    df = eval_dataset.to_pandas()
+    rag_dataset = []
+    for _, row in df.iterrows():
+        rag_dataset.append(
+            {
+                "question": row["question"],
+                "answer": row["answer"],
+                "contexts": ["First aid medical references"],
+                "ground_truths": [row["ground_truth"]],
+                "reference": row["context"]
+            }
+        )
+    rag_df = pd.DataFrame(rag_dataset)
+    return Dataset.from_pandas(rag_df)
+
+def evaluate_ragas_dataset(ragas_dataset):
+    """Run RAGAS evaluation with proper handling of required_columns."""
+    try:
+        result = evaluate(
+            ragas_dataset,
+            metrics=[
+                faithfulness,
+                answer_relevancy,
+                context_recall,
+                answer_correctness,
+                answer_similarity
+            ],
+        )
+        return result
+    except Exception as e:
+        print("⚠️ RAGAS Error:", e)
+        raise e
+
+# Create ground truth dataset
+ground_truth_qac_set = pd.DataFrame({
+    "question": questions,
+    "answer": responses,
+    "context": ["First aid medical references"] * len(questions),
+    "ground_truth": [str(response) for response in ground_truth_responses],
+    "reference": ["First aid medical references"] * len(questions)
+})
+
+eval_dataset = Dataset.from_pandas(ground_truth_qac_set.astype(str))
+
+# Save evaluation datasets
+eval_dataset.to_csv("groundtruth_eval_dataset.csv")
+basic_qa_ragas_dataset = create_ragas_dataset(eval_dataset)
+basic_qa_ragas_dataset.to_csv("basic_qa_ragas_dataset.csv")
+
+# Run evaluation
+basic_qa_result = evaluate_ragas_dataset(basic_qa_ragas_dataset)
+
+print("\n===== Evaluation Results =====")
+print(basic_qa_result)
+
+evaluation_results = basic_qa_result.to_pandas()
+
+# Save evaluation results as log
+
+# ---- STEP 6: Log Results to WandB ----
+
+import wandb
+import pandas as pd
+
+# ✅ Convert `eval_dataset` (Dataset) to Pandas DataFrame
+eval_df = eval_dataset.to_pandas()
+
+# ✅ Convert `basic_qa_ragas_dataset` (Dataset) to Pandas DataFrame
+ragas_df = basic_qa_ragas_dataset.to_pandas()
+
+# ✅ Save DataFrames as CSV
+eval_df.to_csv("groundtruth_eval_dataset.csv", index=False)
+ragas_df.to_csv("basic_qa_ragas_dataset.csv", index=False)
+
+# ✅ Initialize WandB
+wandb.init(
+    project="first-aid-tutor",
+    entity="drsyedfaizan1987-northeastern-university",
+    name="ragas_evaluation",
+    notes="Logging evaluation datasets for first-aid chatbot.",
+    tags=["first-aid", "evaluation", "ragas"]
+)
+
+# ✅ Log DataFrames to WandB as Tables
+wandb.log({"basic_qa_ragas_dataset": wandb.Table(dataframe=evaluation_results)})
+wandb.log({"groundtruth_eval_dataset": wandb.Table(dataframe=eval_df)})
+wandb.log({"basic_qa_ragas_dataset": wandb.Table(dataframe=ragas_df)})
+
+# ✅ Finish WandB run
+wandb.finish()