from openai import OpenAI def evaluate(query: str, answer: str, context: str, api_key: str) -> dict: client = OpenAI(api_key=api_key) def score(prompt: str) -> float: try: r = client.chat.completions.create( model="gpt-3.5-turbo", messages=[{"role": "user", "content": prompt}], temperature=0, max_tokens=5, ) text = r.choices[0].message.content.strip() val = float("".join(c for c in text if c.isdigit() or c == ".")) return min(max(round(val, 2), 0.0), 1.0) except Exception: return 0.5 faithfulness_prompt = f"""Rate how faithful this answer is to the context on a scale of 0.0 to 1.0. 1.0 = every claim in the answer is directly supported by the context. 0.0 = the answer contains claims not present in the context. Return ONLY a number between 0.0 and 1.0. Context: {context[:1500]} Answer: {answer} Score:""" relevance_prompt = f"""Rate how relevant this answer is to the question on a scale of 0.0 to 1.0. 1.0 = the answer directly and completely addresses the question. 0.0 = the answer does not address the question at all. Return ONLY a number between 0.0 and 1.0. Question: {query} Answer: {answer} Score:""" groundedness_prompt = f"""Rate how well-grounded this answer is — does it avoid speculation and stick to facts from the context? Scale 0.0 to 1.0. Return ONLY a number. Context: {context[:1500]} Answer: {answer} Score:""" completeness_prompt = f"""Rate how completely this answer addresses all aspects of the question. Scale 0.0 to 1.0. Return ONLY a number. Question: {query} Answer: {answer} Score:""" return { "faithfulness": score(faithfulness_prompt), "relevance": score(relevance_prompt), "groundedness": score(groundedness_prompt), "completeness": score(completeness_prompt), }