src/evaluators/context_recall_evaluator.py

from sentence_transformers import SentenceTransformer
from sklearn.metrics.pairwise import cosine_similarity

model = None

def evaluate_context_recall(question: str, retrieved_contexts: list) -> dict:
    global model
    if model is None:
        model = SentenceTransformer("all-MiniLM-L6-v2")

    if not retrieved_contexts:
        return {
            "score": 0.0,
            "verdict": "No Context Retrieved"
        }

    combined_context = " ".join(retrieved_contexts)

    question_embedding = model.encode([question])
    context_embedding = model.encode([combined_context])

    score = cosine_similarity(question_embedding, context_embedding)[0][0]
    score = round(float(score), 4)

    if score >= 0.6:
        verdict = "High Recall"
    elif score >= 0.35:
        verdict = "Partial Recall"
    else:
        verdict = "Low Recall"

    return {
        "score": score,
        "verdict": verdict
    }