evaluator.py

from openai import OpenAI

def evaluate(query: str, answer: str, context: str, api_key: str) -> dict:
    client = OpenAI(api_key=api_key)

    def score(prompt: str) -> float:
        try:
            r = client.chat.completions.create(
                model="gpt-3.5-turbo",
                messages=[{"role": "user", "content": prompt}],
                temperature=0,
                max_tokens=5,
            )
            text = r.choices[0].message.content.strip()
            val = float("".join(c for c in text if c.isdigit() or c == "."))
            return min(max(round(val, 2), 0.0), 1.0)
        except Exception:
            return 0.5

    faithfulness_prompt = f"""Rate how faithful this answer is to the context on a scale of 0.0 to 1.0.
1.0 = every claim in the answer is directly supported by the context.
0.0 = the answer contains claims not present in the context.
Return ONLY a number between 0.0 and 1.0.

Context: {context[:1500]}
Answer: {answer}

Score:"""

    relevance_prompt = f"""Rate how relevant this answer is to the question on a scale of 0.0 to 1.0.
1.0 = the answer directly and completely addresses the question.
0.0 = the answer does not address the question at all.
Return ONLY a number between 0.0 and 1.0.

Question: {query}
Answer: {answer}

Score:"""

    groundedness_prompt = f"""Rate how well-grounded this answer is — does it avoid speculation and stick to facts from the context?
Scale 0.0 to 1.0. Return ONLY a number.

Context: {context[:1500]}
Answer: {answer}

Score:"""

    completeness_prompt = f"""Rate how completely this answer addresses all aspects of the question.
Scale 0.0 to 1.0. Return ONLY a number.

Question: {query}
Answer: {answer}

Score:"""

    return {
        "faithfulness": score(faithfulness_prompt),
        "relevance": score(relevance_prompt),
        "groundedness": score(groundedness_prompt),
        "completeness": score(completeness_prompt),
    }