Spaces:
Sleeping
Sleeping
File size: 3,909 Bytes
4796bbf | 1 2 3 4 5 6 7 8 9 10 11 12 13 14 15 16 17 18 19 20 21 22 23 24 25 26 27 28 29 30 31 32 33 34 35 36 37 38 39 40 41 42 43 44 45 46 47 48 49 50 51 52 53 54 55 56 57 58 59 60 61 62 63 64 65 66 67 68 69 70 71 72 73 74 75 76 77 78 79 80 81 82 83 84 85 86 87 88 89 90 91 92 93 94 95 96 97 98 99 100 101 102 103 104 105 106 107 108 109 110 111 112 113 114 115 116 117 118 119 120 121 122 123 | """
Lightweight RAGAS style evaluation without the heavy RAGAS dependency.
Implements:
- Faithfulness: are all claims in the answer supported by the context?
- Answer Relevance: does the answer address the question?
- Context Precision: are the retrieved docs actually relevant to the answer?
Each metric uses an LLM judge (GPT-4o) + optional embedding similarity.
"""
import json
import logging
from typing import Optional
from langchain_openai import ChatOpenAI
from langchain_core.messages import HumanMessage
from .config import get_settings
from .models import EvalRequest, EvalResponse
logger = logging.getLogger(__name__)
settings = get_settings()
_eval_llm = ChatOpenAI(
model = "gpt-4o",
temperature=0.0,
openai_api_key=settings.openai_api_key
)
# Faithfulness
_FAITHFULNESS_PROMPT = """\
You are an evaluation judge. Given the CONTEXT and an ANSWER, assess whether every \
factual claim in the answer is explicitly supported by the context.
Context:
{context}
Answer:
{answer}
Score the faithfulness from 0.0 (completely unsupported) to 1.0 (Fully supported).
Respond ONLY with a JSON object: {{"faithfulness": <float>, "reasoning":"<brief>"}}
"""
async def score_faithfulness(answer: str, contexts: list[str]) -> float:
context_str = "\n---\n".join(contexts)
prompt = _FAITHFULNESS_PROMPT.format(context=context_str, answer=answer)
response = await _eval_llm.ainvoke([HumanMessage(content=prompt)])
try:
data = json.loads(response.content)
return float(data["faithfulness"])
except Exception:
logger.warning("Faithfulness parse error")
return 0.0
# Answer Relevance
_RELEVANCE_PROMPT = """\
You are an evaluation judge. Give a Question and an ANSWER, score how well \
the answer addresses the question.
Question: {question}
Answer: {answer}
Score from 0.0 (completely irrelevant) to 1.0 (perfectly answers the question).
Respond ONLY with a JSON object: {{"relevance": <float>, "reasoning": "<brief>"}}
"""
async def score_answer_relevance(question: str, answer: str) -> float:
prompt = _RELEVANCE_PROMPT.format(question=question,answer=answer)
response = await _eval_llm.ainvoke([HumanMessage(content=prompt)])
try:
data = json.loads(response.content)
return float(data["relevance"])
except Exception:
logger.warning("Relevance parse error")
return 0.0
# Context Precision
_PRECISION_PROMPT = """\
You are an evaluation judge. For each retrieved context below, determine whether \
it was USEFUL for answering the question.
Question: {question}
Answer: {answer}
Contexts:
{contexts}
Response with a JSON object:
{{"useful":[trur/false, ...], "precision": <float 0-1>}}
where useful[i] = whether context i contributed to the answer.
"""
async def score_context_precision(
question: str, answer: str, contexts: list[str]
) -> float:
ctx_str = "\n".join(f"[{i+1}] {c[:300]}" for i,c in enumerate(contexts))
prompt = _PRECISION_PROMPT.format(question=question,answer=answer,contexts=ctx_str)
response = await _eval_llm.invoke([HumanMessage(content=prompt)])
try:
data = json.loads(response.content)
return float(data["precision"])
except Exception:
return 0.0
# Composite evaluator
async def evaluate(req: EvalRequest) -> EvalResponse:
faithfulness = await score_faithfulness(req.answer, req.contexts)
relevance = await score_answer_relevance(req.question, req.answer)
precision = await score_context_precision(req.question, req.answer, req.contexts)
passed = (
faithfulness >= settings.faithfullness_threshold
and relevance >= settings.answer_relevance_threshold
)
return EvalResponse(
faithfulness=round(faithfulness,3),
answer_relevance=round(relevance,3),
context_precision=round(precision,3),
passed=passed
)
print("[eval] Module ready") |