Add RAG evaluators - context relevance and recall + updated main

llm-eval-dashboard

@@ -0,0 +1 @@
+Subproject commit 30b0f17dd666e144887bcfe8ebd564aaa537b2fc

main.py

@@ -1,51 +1,52 @@
 from fastapi import FastAPI, HTTPException
 from pydantic import BaseModel
+from typing import List
 from src.aggregator import evaluate_all
 from src.database import init_db, save_evaluation
 
 app = FastAPI(
     title="LLM Evaluation & Hallucination Detection Framework",
-    version="1.0.0"
+    version="2.0.0"
 )
-init_db()
 
+init_db()
 
-# Define what the request should look like
 class EvalRequest(BaseModel):
-    context: str
     question: str
+    retrieved_contexts: List[str]
     llm_response: str
 
-# Define what the response will look like
 class EvalResponse(BaseModel):
     final_verdict: str
-    cosine: dict
-    fluency: dict
-    bert_score: dict
-    nli: dict
+    retrieval_evaluation: dict
+    generation_evaluation: dict
 
 @app.get("/")
 def home():
-    return {"message": "LLM Evaluation Framework is running"}
+    return {"message": "LLM Evaluation Framework v2.0 is running"}
 
 @app.post("/evaluate", response_model=EvalResponse)
 def evaluate(request: EvalRequest):
 
-    # Edge case — empty inputs
-    if not request.context.strip():
-        raise HTTPException(status_code=400, detail="Context cannot be empty")
     if not request.question.strip():
         raise HTTPException(status_code=400, detail="Question cannot be empty")
+    if not request.retrieved_contexts:
+        raise HTTPException(status_code=400, detail="Retrieved contexts cannot be empty")
     if not request.llm_response.strip():
         raise HTTPException(status_code=400, detail="LLM response cannot be empty")
 
-    # Run evaluation
     result = evaluate_all(
-        context=request.context,
         question=request.question,
+        retrieved_contexts=request.retrieved_contexts,
         llm_response=request.llm_response
     )
-    save_evaluation(request.context, request.question, request.llm_response, result)
+
+    save_evaluation(
+        context=" ".join(request.retrieved_contexts),
+        question=request.question,
+        llm_response=request.llm_response,
+        result=result
+    )
 
     return result
 
@@ -54,16 +55,13 @@ from src.database import get_all_evaluations
 @app.get("/history")
 def history():
     rows = get_all_evaluations()
-    
     results = []
     for row in rows:
         results.append({
             "id": row[0],
-            "context": row[1],
             "question": row[2],
             "llm_response": row[3],
             "final_verdict": row[4],
             "created_at": row[11]
         })
-    
     return {"total": len(results), "evaluations": results}

src/evaluators/context_recall_evaluator.py

@@ -0,0 +1,35 @@
+from sentence_transformers import SentenceTransformer
+from sklearn.metrics.pairwise import cosine_similarity
+
+model = None
+
+def evaluate_context_recall(question: str, retrieved_contexts: list) -> dict:
+    global model
+    if model is None:
+        model = SentenceTransformer("all-MiniLM-L6-v2")
+
+    if not retrieved_contexts:
+        return {
+            "score": 0.0,
+            "verdict": "No Context Retrieved"
+        }
+
+    combined_context = " ".join(retrieved_contexts)
+
+    question_embedding = model.encode([question])
+    context_embedding = model.encode([combined_context])
+
+    score = cosine_similarity(question_embedding, context_embedding)[0][0]
+    score = round(float(score), 4)
+
+    if score >= 0.6:
+        verdict = "High Recall"
+    elif score >= 0.35:
+        verdict = "Partial Recall"
+    else:
+        verdict = "Low Recall"
+
+    return {
+        "score": score,
+        "verdict": verdict
+    }

src/evaluators/context_relevance_evaluator.py

@@ -0,0 +1,40 @@
+from sentence_transformers import SentenceTransformer
+from sklearn.metrics.pairwise import cosine_similarity
+import numpy as np
+
+model = None
+
+def evaluate_context_relevance(question: str, retrieved_contexts: list) -> dict:
+    global model
+    if model is None:
+        model = SentenceTransformer("all-MiniLM-L6-v2")
+
+    if not retrieved_contexts:
+        return {
+            "scores": [],
+            "average_score": 0.0,
+            "verdict": "No Context Retrieved"
+        }
+
+    question_embedding = model.encode([question])
+
+    scores = []
+    for chunk in retrieved_contexts:
+        chunk_embedding = model.encode([chunk])
+        score = cosine_similarity(question_embedding, chunk_embedding)[0][0]
+        scores.append(round(float(score), 4))
+
+    average_score = round(float(np.mean(scores)), 4)
+
+    if average_score >= 0.6:
+        verdict = "Highly Relevant Context"
+    elif average_score >= 0.4:
+        verdict = "Partially Relevant Context"
+    else:
+        verdict = "Irrelevant Context Retrieved"
+
+    return {
+        "scores": scores,
+        "average_score": average_score,
+        "verdict": verdict
+    }