eval.py

import os
from dotenv import load_dotenv
from gradio_client import Client
from datasets import Dataset
from ragas import evaluate
from ragas.metrics import (
    answer_relevancy,
    answer_correctness,
    answer_similarity
)
import pandas as pd
import wandb

# Load API Key from .env file
load_dotenv()
api_key = os.getenv("OPENAI_API_KEY")

# Initialize Gradio Client
client = Client("DrSyedFaizan/First_Aid_Assistant")

# Questions and Ground Truth Answers
questions = [
    "What are the first aid measures for high fever in infants?",
    "What are the signs and symptoms of low blood sugar?",
    "What does RICE stand for in first aid treatment?",
    "What is the first aid treatment of bleeding?",
    "What is the first aid management of burns?",
    "What are the signs and symptoms of stroke?",
    "What is the treatment of snake bite?",
    "How do you provide first aid for choking?",
    "What are the immediate steps to treat a fainting patient?",
    "What are the First aid measures for taking care of a patient with insect stings and animal bites?"
]

ground_truth_responses = [
    "For infants aged 0-3 months with a rectal temperature of 100.4oF (38oC) or higher, refer to the doctor even if the child does not have any other signs or symptoms. For infants aged 3-6 months with a rectal temperature up to 102oF (38.9oC), encourage the child to rest and drink plenty of fluids without the need for medication. If the child seems unusually irritable, lethargic, or uncomfortable, refer to the doctor.",
    "Irritability, confusion, tremors, blurring of vision, coma, seizures, tachycardia, hypotension, and cold and clammy skin are signs and symptoms of low blood sugar.",
    "RICE stands for: R: Rest the injured part, I: Ice the area, C: Compress with a bandage, E: Elevate the injured part to divert the blood flow away from the area",
    "Control bleeding with direct pressure, cover wound with sterile dressing or clean cloth, diaper, or sanitary napkin, place gloved hand over dressing, press firmly, continue pressure until bleeding stops. DO NOT remove dressing. If soaked through, add more material and continue pressure. If bleeding does not slow after 5 minutes of direct pressure, have victim lie down, continue direct pressure on wound, and apply pressure to pressure point between wound and heart.",
    "Douse the burnt area with cool water within 10 minutes of injury. This may halt burn process in the tissues which may minimise burn depth and reduce pain considerably. Drink plenty of water and remove jewellery from the burned area.",
    "Signs and symptoms of a stroke include a sudden feeling of weakness or numbness of the face, arm, or leg on one side of the body, loss of vision or dimming in one or both eyes, loss of speech, difficulty in talking or understanding what others are saying, and a sudden, severe headache with no known cause.",
    "The treatment of a snake bite includes: a) Lay the patient down. b) Give him complete rest. c) Calm and reassure him. d) Do not make him walk. e) Tie immediately a piece of cloth or a tourniquet, tightly above the bite to prevent the venous blood return. It should be loosened for a few seconds at a regular interval of about 10 minutes. f) Wash cuts gently with normal saline or antiseptic lotion if available otherwise with soapy water. Apply a clean dressing. Immobilize the affected limb. Apply Ice packs on the wound. Shift the patient to the hospital immediately. Take the killed snake, if available for identification.",
    "To provide first aid for choking in an adult: Stand behind the victim and wrap your arms around their waist. Grasp your fist with your other hand and place your thumb against the abdomen slightly above the navel and below the rib cage. Perform a quick upward thrust on the abdomen with your fist. Repeat this several times until the foreign body is expelled.",
    "Immediate steps to treat a fainting patient include making the person lie flat for at least 15 minutes or sit if lightheadedness is present, opening the airway and assessing for breathing, raising the person's legs above the level of their heart if breathing, loosening any restrictive clothing around the neck or waist, checking for breathing and heart rate if the person remains unconscious, and calling an ambulance for immediate assessment and treatment if needed.",
    "The first aid measures for taking care of a patient with insect stings are: 1) Apply pressure with a clean bandage or towel to stop bleeding. 2) Clean and scrap the area to remove carefully the wings of the insect. 3) Give immediately a dose of tetanus toxide. 4) Apply antihistamine ointment to reduce itching, swelling, and pain."
]


references = list(ground_truth_responses)

# Fetch Responses from the Chatbot
print("\nFetching Responses from the Chatbot...\n")
bot_responses = []

for question in questions:
    try:
        result = client.predict(chatbot=[], message=question, api_name="/respond")
        chat_history = result[1]
        response = next((entry["content"] for entry in chat_history if entry["role"] == "assistant"), "[NO RESPONSE]")
    except Exception as e:
        response = f"[ERROR: {e}]"
    
    bot_responses.append(response)

# ---- Create Dataset for RAGAS ----
ground_truth_qac_set = pd.DataFrame({
    "question": questions,
    "answer": bot_responses,
    "context": [[ref] for ref in references],            # ✅ Wrap each context in a list
    "ground_truth": ground_truth_responses,
    "reference": [[ref] for ref in references],          # ✅ Wrap each reference in a list
      # ✅ Wrap each retrieved_context in a list
})

eval_dataset = Dataset.from_pandas(ground_truth_qac_set.astype(str))

# ---- RAGAS Evaluation ----
def evaluate_ragas_dataset(ragas_dataset):
    try:
        result = evaluate(
            ragas_dataset,
            metrics=[
        
                answer_relevancy,
                answer_correctness,
                answer_similarity
            ],
        )
        return result
    except Exception as e:
        print("⚠️ RAGAS Error:", e)
        raise e

# Run Evaluation
basic_qa_ragas_dataset = eval_dataset
basic_qa_result = evaluate_ragas_dataset(basic_qa_ragas_dataset)

# Save Evaluation Results
evaluation_results = basic_qa_result.to_pandas()
evaluation_results.to_html("ragas_evaluation_results.html", index=False)

# ---- Log Results to WandB ----
wandb.init(
    project="first-aid-tutor",
    entity="drsyedfaizan1987-northeastern-university",
    name="ragas_evaluation",
    notes="Logging evaluation datasets for first-aid chatbot.",
    tags=["first-aid", "evaluation", "ragas"]
)

wandb.log({"ragas_evaluation_results": wandb.Table(dataframe=evaluation_results)})
wandb.finish()

print("\n===== ✅ Evaluation Results Saved to 'ragas_evaluation_results.html' =====")