src/evaluators/context_relevance_evaluator.py

from sentence_transformers import SentenceTransformer
from sklearn.metrics.pairwise import cosine_similarity
import numpy as np

model = None

def evaluate_context_relevance(question: str, retrieved_contexts: list) -> dict:
    global model
    if model is None:
        model = SentenceTransformer("all-MiniLM-L6-v2")

    if not retrieved_contexts:
        return {
            "scores": [],
            "average_score": 0.0,
            "verdict": "No Context Retrieved"
        }

    question_embedding = model.encode([question])

    scores = []
    for chunk in retrieved_contexts:
        chunk_embedding = model.encode([chunk])
        score = cosine_similarity(question_embedding, chunk_embedding)[0][0]
        scores.append(round(float(score), 4))

    average_score = round(float(np.mean(scores)), 4)

    if average_score >= 0.6:
        verdict = "Highly Relevant Context"
    elif average_score >= 0.4:
        verdict = "Partially Relevant Context"
    else:
        verdict = "Irrelevant Context Retrieved"

    return {
        "scores": scores,
        "average_score": average_score,
        "verdict": verdict
    }