Create evaluation.py

evaluation.py

@@ -0,0 +1,126 @@
+from llama_index.core.schema import MetadataMode, TextNode
+from llama_index.core.evaluation import EmbeddingQAFinetuneDataset
+from tqdm import tqdm
+from typing import Dict, List
+import uuid
+import time
+from llama_index.llms.openai import OpenAI
+import pandas as pd
+
+
+def evaluate_faithfulness(
+    question: str,
+    answer: str,
+    contexts: list[str],
+    llm,
+) -> float:
+    context_text = "\n\n".join(contexts)
+
+    prompt = f"""
+You are an evaluator.
+
+Question:
+{question}
+
+Answer:
+{answer}
+
+Retrieved Context:
+{context_text}
+
+Task:
+Determine whether the answer is fully supported by the retrieved context.
+
+Scoring:
+- 1.0 → All claims are supported by the context
+- 0.5 → Some claims supported, some not
+- 0.0 → Mostly or fully unsupported / hallucinated
+
+Return ONLY the score (1.0, 0.5, or 0.0).
+"""
+
+    response = llm.complete(prompt)
+    try:
+        return float(str(response).strip())
+    except ValueError:
+        return 0.0
+
+def evaluate_answer_relevance(
+    question: str,
+    answer: str,
+    llm,
+) -> float:
+    prompt = f"""
+You are an evaluator.
+
+Question:
+{question}
+
+Answer:
+{answer}
+
+Task:
+Evaluate how well the answer addresses the question.
+
+Scoring:
+- 1.0 → Fully answers the question
+- 0.5 → Partially answers
+- 0.0 → Does not answer / off-topic
+
+Return ONLY the score (1.0, 0.5, or 0.0).
+"""
+
+    response = llm.complete(prompt)
+    try:
+        return float(str(response).strip())
+    except ValueError:
+        return 0.0
+
+def evaluate_rag_answers_safe(
+    queries: list[str],
+    index,
+    llm,
+    top_k: int = 10,
+    per_call_delay: float = 6.5  # 6.5 seconds between Cohere API calls
+):
+    """
+    Evaluate RAG answers safely with respect to Cohere trial key limits.
+    """
+
+    rows = []
+    query_engine = index.as_query_engine(
+        similarity_top_k=top_k,
+        node_postprocessors=[cohere_rerank3],  # optional
+    )
+
+    for query in tqdm(queries, desc="Evaluating queries"):
+        response = query_engine.query(query)
+        answer = response.response
+        contexts = [n.node.get_content() for n in response.source_nodes]
+
+        faithfulness = evaluate_faithfulness(
+            question=query,
+            answer=answer,
+            contexts=contexts,
+            llm=llm,
+        )
+
+        relevance = evaluate_answer_relevance(
+            question=query,
+            answer=answer,
+            llm=llm,
+        )
+
+        rows.append({
+            "query": query,
+            "faithfulness": faithfulness,
+            "answer_relevance": relevance,
+        })
+
+        # Sleep after each call to avoid hitting the 10/min trial limit
+        time.sleep(per_call_delay)
+
+    df = pd.DataFrame(rows)
+    print("Average Scores:")
+    print(df.mean(numeric_only=True))
+    return df