Separate LLM and RAG evaluation

llm-eval-dashboard

@@ -1 +1 @@
-Subproject commit 1e3cca61e97ba9dcc2fc76eab40e0e0c7ed687a2
+Subproject commit 5e1c5e3efe5a71746d0a07e35226b77caef3d743

main.py

@@ -1,9 +1,9 @@
 from fastapi import FastAPI, HTTPException
 from pydantic import BaseModel
 from typing import List
-from src.aggregator import evaluate_all
 from src.database import init_db, save_evaluation
 import traceback
+from src.aggregator import evaluate_all, evaluate_generation_only
 
 app = FastAPI(
     title="LLM Evaluation & Hallucination Detection Framework",
@@ -20,6 +20,11 @@ class EvalResponse(BaseModel):
     retrieval_evaluation: dict
     generation_evaluation: dict
 
+class LLMOnlyEvalRequest(BaseModel):
+    question: str
+    context: str
+    llm_response: str
+
 @app.post("/evaluate", response_model=EvalResponse)
 def evaluate(request: EvalRequest):
     if not request.question.strip():
@@ -47,6 +52,51 @@ def evaluate(request: EvalRequest):
 
 init_db()
 
+@app.post("/evaluate-llm")
+def evaluate_llm(request: LLMOnlyEvalRequest):
+
+    if not request.question.strip():
+        raise HTTPException(
+            status_code=400,
+            detail="Question cannot be empty"
+        )
+
+    if not request.context.strip():
+        raise HTTPException(
+            status_code=400,
+            detail="Context cannot be empty"
+        )
+
+    if not request.llm_response.strip():
+        raise HTTPException(
+            status_code=400,
+            detail="LLM response cannot be empty"
+        )
+
+    try:
+
+        result = evaluate_generation_only(
+            question=request.question,
+            context=request.context,
+            llm_response=request.llm_response
+        )
+
+        save_evaluation(
+            context=request.context,
+            question=request.question,
+            llm_response=request.llm_response,
+            result=result
+        )
+
+        return result
+
+    except Exception as e:
+
+        raise HTTPException(
+            status_code=500,
+            detail=str(e) + "\n" + traceback.format_exc()
+        )
+
 
 @app.get("/")
 def home():

src/aggregator.py

@@ -41,4 +41,58 @@ def evaluate_all(question: str, retrieved_contexts: list, llm_response: str) ->
             "bert_score": bert_result,
             "nli": nli_result
         }
+    }
+
+def evaluate_generation_only(
+    question: str,
+    context: str,
+    llm_response: str
+) -> dict:
+
+    cosine_result = evaluate_cosine(
+        question,
+        llm_response
+    )
+
+    fluency_result = evaluate_fluency(
+        llm_response
+    )
+
+    bert_result = evaluate_bert_score(
+        context,
+        llm_response
+    )
+
+    nli_result = evaluate_nli(
+        context,
+        llm_response
+    )
+
+    # Generation-only final verdict
+    if nli_result["verdict"] == "Hallucinated":
+        final_verdict = "Hallucinated"
+
+    elif (
+        nli_result["verdict"] == "Faithful"
+        and bert_result["score"] >= 0.70
+    ):
+        final_verdict = "Faithful"
+
+    elif (
+        cosine_result["verdict"] == "Irrelevant"
+        and len(llm_response.split()) > 2
+    ):
+        final_verdict = "Irrelevant"
+
+    else:
+        final_verdict = "Unverifiable"
+
+    return {
+        "final_verdict": final_verdict,
+        "generation_evaluation": {
+            "cosine": cosine_result,
+            "fluency": fluency_result,
+            "bert_score": bert_result,
+            "nli": nli_result
+        }
     }