community_contributions/chatbot_rag_evaluation/evaluator.py

from pydantic import BaseModel
from openai import OpenAI
import os
from dotenv import load_dotenv


MODEL = "gemini-2.0-flash"

class Evaluation(BaseModel):
    is_acceptable: bool
    feedback: str


class Evaluator:
    def __init__(self, name="", model=MODEL):
        load_dotenv(override=True)
        google_api_key = os.getenv('GOOGLE_API_KEY')

        self.name=name
        self.model=model
        self._gemini = OpenAI(api_key=google_api_key, base_url="https://generativelanguage.googleapis.com/v1beta/openai/")

    def _evaluator_system_prompt(self):
        return f"You are an evaluator that decides whether a response to a question is acceptable. \
            You are provided with a conversation between a User and an Agent. Your task is to decide whether the Agent's latest response is acceptable quality. \
            The Agent is playing the role of {self.name} and is representing {self.name} on their website. \
            The Agent has been instructed to be professional and engaging, as if talking to a potential client or future employer who came across the website. \
            The Agent has been provided with context on {self.name} in the form of their summary, experience and CV. \
            With this context, please evaluate the latest response, replying with whether the response is acceptable and your feedback."

    def _evaluator_user_prompt(self, reply, message, history):
        user_prompt = f"Here's the conversation between the User and the Agent: \n\n{history}\n\n"
        user_prompt += f"Here's the latest message from the User: \n\n{message}\n\n"
        user_prompt += f"Here's the latest response from the Agent: \n\n{reply}\n\n"
        user_prompt += "Please evaluate the response, replying with whether it is acceptable and your feedback."
        return user_prompt
    
    def evaluate(self, reply, message, history) -> Evaluation:
        messages = [{"role": "system", "content": self._evaluator_system_prompt()}] + [{"role": "user", "content": self._evaluator_user_prompt(reply, message, history)}]
        response = self._gemini.beta.chat.completions.parse(model=self.model, messages=messages, response_format=Evaluation)
        return response.choices[0].message.parsed