File size: 3,995 Bytes
8446976 a165192 c17e20a 8446976 c17e20a 8446976 | 1 2 3 4 5 6 7 8 9 10 11 12 13 14 15 16 17 18 19 20 21 22 23 24 25 26 27 28 29 30 31 32 33 34 35 36 37 38 39 40 41 42 43 44 45 46 47 48 49 50 51 52 53 54 55 56 57 58 59 60 61 62 63 64 65 66 67 68 69 70 71 72 73 74 75 76 77 78 79 80 81 82 83 84 85 86 87 88 89 90 91 92 93 94 95 96 97 98 99 100 101 102 103 104 105 106 107 108 109 110 111 112 113 114 115 116 117 | import os
from dotenv import load_dotenv
from server.utils import load_config, setup_logger
load_dotenv()
logger = setup_logger(__name__)
def _safe_round(val, decimals: int = 4):
try:
return round(float(val), decimals)
except (TypeError, ValueError):
return None
def run_ragas_eval(eval_log: list[dict], n_pairs: int = 10) -> dict:
"""
Run RAGAS on last n_pairs from session eval_log.
Computes faithfulness + answer_relevancy (no ground_truth required).
context_precision + context_recall return null — require labeled ground_truth dataset.
Args:
eval_log: list of {query, answer, contexts, ...} dicts from session
n_pairs: number of recent pairs to evaluate
Returns:
dict with faithfulness, answer_relevancy, context_precision (null),
context_recall (null), per_query, sample_count
"""
# Lazy imports — ragas 0.1.x pulls langchain_community.chat_models.vertexai at module
# level, removed in langchain-community 0.3.x. Pin ragas>=0.2.0 in requirements.txt.
try:
from ragas import evaluate, EvaluationDataset, SingleTurnSample
from ragas.metrics import Faithfulness, AnswerRelevancy
from ragas.llms import LangchainLLMWrapper
from ragas.embeddings import LangchainEmbeddingsWrapper
except (ImportError, ModuleNotFoundError) as e:
logger.error("RAGAS import failed (dependency conflict): %s", e)
return {
"faithfulness": None,
"answer_relevancy": None,
"context_precision": None,
"context_recall": None,
"per_query": [],
"sample_count": 0,
"error": f"RAGAS unavailable: {e}. Pin ragas>=0.2.0,<0.3.0 in requirements.txt.",
}
from langchain_groq import ChatGroq
from langchain_openai import OpenAIEmbeddings
pairs = [p for p in eval_log if p.get("contexts")]
pairs = pairs[-n_pairs:]
if not pairs:
return {
"faithfulness": None,
"answer_relevancy": None,
"context_precision": None,
"context_recall": None,
"per_query": [],
"sample_count": 0,
"note": "No session pairs with contexts found. Ask questions first.",
}
config = load_config()
llm_cfg = config.get("llm", {})
ragas_llm = LangchainLLMWrapper(
ChatGroq(model=llm_cfg["model"], api_key=os.getenv("GROQ_API_KEY", ""), temperature=0.0)
)
# Groq has no embeddings endpoint — Euron API handles embeddings
ragas_emb = LangchainEmbeddingsWrapper(
OpenAIEmbeddings(
model="text-embedding-3-small",
openai_api_key=os.getenv("EURON_API_KEY", ""),
openai_api_base="https://api.euron.one/api/v1/euri",
)
)
samples = [
SingleTurnSample(
user_input=p["query"],
response=p["answer"],
retrieved_contexts=p["contexts"],
)
for p in pairs
]
dataset = EvaluationDataset(samples=samples)
metrics = [
Faithfulness(llm=ragas_llm),
AnswerRelevancy(llm=ragas_llm, embeddings=ragas_emb),
]
logger.info(f"Running RAGAS on {len(samples)} pairs")
results = evaluate(dataset=dataset, metrics=metrics)
scores_df = results.to_pandas()
per_query = []
for i, row in scores_df.iterrows():
per_query.append({
"query": pairs[i]["query"],
"faithfulness": _safe_round(row.get("faithfulness")),
"answer_relevancy": _safe_round(row.get("answer_relevancy")),
})
return {
"faithfulness": _safe_round(results["faithfulness"]),
"answer_relevancy": _safe_round(results["answer_relevancy"]),
"context_precision": None,
"context_recall": None,
"per_query": per_query,
"sample_count": len(samples),
"note": "context_precision and context_recall require labeled ground_truth dataset",
}
|