server/eval/ragas_eval.py

import os
from dotenv import load_dotenv

from server.utils import load_config, setup_logger

load_dotenv()
logger = setup_logger(__name__)


def _safe_round(val, decimals: int = 4):
    try:
        return round(float(val), decimals)
    except (TypeError, ValueError):
        return None


def run_ragas_eval(eval_log: list[dict], n_pairs: int = 10) -> dict:
    """
    Run RAGAS on last n_pairs from session eval_log.

    Computes faithfulness + answer_relevancy (no ground_truth required).
    context_precision + context_recall return null — require labeled ground_truth dataset.

    Args:
        eval_log: list of {query, answer, contexts, ...} dicts from session
        n_pairs: number of recent pairs to evaluate

    Returns:
        dict with faithfulness, answer_relevancy, context_precision (null),
        context_recall (null), per_query, sample_count
    """
    # Lazy imports — ragas 0.1.x pulls langchain_community.chat_models.vertexai at module
    # level, removed in langchain-community 0.3.x. Pin ragas>=0.2.0 in requirements.txt.
    try:
        from ragas import evaluate, EvaluationDataset, SingleTurnSample
        from ragas.metrics import Faithfulness, AnswerRelevancy
        from ragas.llms import LangchainLLMWrapper
        from ragas.embeddings import LangchainEmbeddingsWrapper
    except (ImportError, ModuleNotFoundError) as e:
        logger.error("RAGAS import failed (dependency conflict): %s", e)
        return {
            "faithfulness": None,
            "answer_relevancy": None,
            "context_precision": None,
            "context_recall": None,
            "per_query": [],
            "sample_count": 0,
            "error": f"RAGAS unavailable: {e}. Pin ragas>=0.2.0,<0.3.0 in requirements.txt.",
        }
    from langchain_groq import ChatGroq
    from langchain_openai import OpenAIEmbeddings

    pairs = [p for p in eval_log if p.get("contexts")]
    pairs = pairs[-n_pairs:]

    if not pairs:
        return {
            "faithfulness": None,
            "answer_relevancy": None,
            "context_precision": None,
            "context_recall": None,
            "per_query": [],
            "sample_count": 0,
            "note": "No session pairs with contexts found. Ask questions first.",
        }

    config = load_config()
    llm_cfg = config.get("llm", {})
    ragas_llm = LangchainLLMWrapper(
        ChatGroq(model=llm_cfg["model"], api_key=os.getenv("GROQ_API_KEY", ""), temperature=0.0)
    )
    # Groq has no embeddings endpoint — Euron API handles embeddings
    ragas_emb = LangchainEmbeddingsWrapper(
        OpenAIEmbeddings(
            model="text-embedding-3-small",
            openai_api_key=os.getenv("EURON_API_KEY", ""),
            openai_api_base="https://api.euron.one/api/v1/euri",
        )
    )

    samples = [
        SingleTurnSample(
            user_input=p["query"],
            response=p["answer"],
            retrieved_contexts=p["contexts"],
        )
        for p in pairs
    ]

    dataset = EvaluationDataset(samples=samples)
    metrics = [
        Faithfulness(llm=ragas_llm),
        AnswerRelevancy(llm=ragas_llm, embeddings=ragas_emb),
    ]

    logger.info(f"Running RAGAS on {len(samples)} pairs")
    results = evaluate(dataset=dataset, metrics=metrics)

    scores_df = results.to_pandas()
    per_query = []
    for i, row in scores_df.iterrows():
        per_query.append({
            "query": pairs[i]["query"],
            "faithfulness": _safe_round(row.get("faithfulness")),
            "answer_relevancy": _safe_round(row.get("answer_relevancy")),
        })

    return {
        "faithfulness": _safe_round(results["faithfulness"]),
        "answer_relevancy": _safe_round(results["answer_relevancy"]),
        "context_precision": None,
        "context_recall": None,
        "per_query": per_query,
        "sample_count": len(samples),
        "note": "context_precision and context_recall require labeled ground_truth dataset",
    }