src/eval/relevancy.py

from src.llm.llm_client import LLMClient

class RelevancyGrader:
    def __init__(self, llm_client: LLMClient):
        self.llm = llm_client

    def grade(self, query: str, answer: str) -> dict:
        """
        Returns score (0-1) on whether the answer addresses the query.
        """
        system_prompt = "You are a grader assessing if a generated answer is relevant to the user query."
        user_prompt = f"""
        User Query: {query}
        Generated Answer: {answer}
        
        Does the answer directly address the query?
        Give a score between 0 and 1, and a boolean label (true/false).
        Return JSON format: {{"score": 0.9, "relevant": true}}
        """
        
        messages = [
            {"role": "system", "content": system_prompt},
            {"role": "user", "content": user_prompt}
        ]
        
        try:
            response = self.llm.chat(messages, response_format={"type": "json_object"})
            import json
            data = json.loads(response)
            # print(f"DEBUG_RELEVANCY_RAW: {response}")
            return data
        except Exception as e:
            print(f"DEBUG_RELEVANCY_ERROR: {e}")
            print(f"DEBUG_RELEVANCY_RESPONSE_WAS: {locals().get('response', 'Not generated')}")
            return {"score": 0.5, "relevant": False, "error": str(e)}