src/evaluate_rag.py

import os
import sys
import json
import time
import urllib.request
import urllib.error
from typing import List, Dict

# Reconfigure stdout for UTF-8 to support Windows console emoji printing
if sys.platform.startswith("win"):
    import io
    sys.stdout = io.TextIOWrapper(sys.stdout.buffer, encoding='utf-8')

# Add the current directory to sys.path so we can import from app
sys.path.append(os.path.dirname(os.path.abspath(__file__)))

try:
    from app import answer_question, search_archives
except ImportError as e:
    print(f"Error importing app backend: {e}")
    sys.exit(1)

# ==============================================================================
# GOLDEN EVALUATION DATASET
# ==============================================================================
# A representative set of critical doctrinal questions to run automated metrics on.
GOLDEN_DATASET = [
    {
        "query": "what are the seven seals?",
        "expected_topics": ["mysteries of God", "Lamb opening the seals", "white horse", "red horse"]
    },
    {
        "query": "what are the seven thunders?",
        "expected_topics": ["mysteries", "unwritten", "seventh seal", "revelation"]
    },
    {
        "query": "What is the relationship between the Serpent's Seed and Cain?",
        "expected_topics": ["serpent seed", "Eve", "Cain", "lineage", "Satan"]
    },
    {
        "query": "Summarize the seven church ages.",
        "expected_topics": ["Ephesus", "Smyrna", "Pergamos", "Thyatira", "Sardis", "Philadelphia", "Laodicea"]
    }
]

# ==============================================================================
# LLM-AS-A-JUDGE METRIC EVALUATORS
# ==============================================================================
def call_gemini_judge(prompt: str) -> str:
    """Helper to query the Gemini API to act as an objective evaluation judge with automatic retry backoff."""
    api_key = os.getenv("GOOGLE_API_KEY")
    if not api_key:
        return "ERROR: GOOGLE_API_KEY not set."
        
    url = f"https://generativelanguage.googleapis.com/v1beta/models/gemini-pro:generateContent?key={api_key}"
    headers = {"Content-Type": "application/json"}
    payload = {
        "contents": [{"parts": [{"text": prompt}]}],
        "generationConfig": {"temperature": 0.0}  # Temp 0.0 for deterministic evaluation scoring!
    }
    
    max_retries = 5
    backoff = 10.0
    
    for attempt in range(max_retries):
        req = urllib.request.Request(
            url,
            data=json.dumps(payload).encode("utf-8"),
            headers=headers,
            method="POST"
        )
        try:
            with urllib.request.urlopen(req) as response:
                res = json.loads(response.read().decode("utf-8"))
                return res["candidates"][0]["content"]["parts"][0]["text"].strip()
        except urllib.error.HTTPError as e:
            if e.code == 429:
                print(f"   [RATE LIMIT 429] Judge hit Gemini rate limits. Retrying in {backoff}s (Attempt {attempt+1}/{max_retries})...")
                time.sleep(backoff)
                backoff *= 2.0
            else:
                return f"ERROR: judge API call failed with HTTP {e.code}: {e.reason}"
        except Exception as e:
            return f"ERROR: judge API call failed: {e}"
            
    return "ERROR: Exceeded maximum API retry limit due to continuous 429 Rate Limits."

def evaluate_faithfulness(answer: str, retrieved_contexts: List[str]) -> float:
    """Evaluates Groundedness/Faithfulness: Is the answer supported *only* by the context?"""
    context_text = "\n---\n".join(retrieved_contexts)
    prompt = f"""
You are an expert AI system evaluation judge. Your task is to score the FAITHFULNESS (groundedness) of a generated answer based ONLY on the provided retrieved context paragraphs.

RETRIVED CONTEXT:
{context_text}

GENERATED ANSWER:
{answer}

INSTRUCTIONS:
- Analyze the generated answer. Identify every factual claim made in it.
- Check if each claim is directly supported by the retrieved context.
- Rate the FAITHFULNESS on a scale of 0.0 to 1.0:
  * 1.0: All claims in the answer are fully grounded and supported by the context.
  * 0.7 - 0.9: Most claims are supported, but there are minor ungrounded additions or extensions.
  * 0.4 - 0.6: Significant claims in the answer are not found in the context (hallucinations).
  * 0.0 - 0.3: The answer has no grounding in the context or directly contradicts it.

Respond with EXACTLY a single floating-point number representing the score. Do not write any other text or reasoning.
"""
    result = call_gemini_judge(prompt)
    try:
        return float(result.strip())
    except ValueError:
        # Fallback parsing
        for token in result.split():
            try:
                return float(token)
            except ValueError:
                continue
        return 0.5

def evaluate_answer_relevance(query: str, answer: str) -> float:
    """Evaluates Answer Relevance: Does the generated answer address the user query?"""
    prompt = f"""
You are an expert AI system evaluation judge. Your task is to score the ANSWER RELEVANCE of a generated response based on the user's initial query.

USER QUERY:
{query}

GENERATED ANSWER:
{answer}

INSTRUCTIONS:
- Score how directly the answer addresses the user query.
- Rate the RELEVANCE on a scale of 0.0 to 1.0:
  * 1.0: The answer directly, fully, and clearly answers the user query.
  * 0.7 - 0.9: The answer addresses the query, but misses a small aspect or includes unnecessary extra details.
  * 0.4 - 0.6: The answer is partially relevant but misses the core question or is vague.
  * 0.0 - 0.3: The answer is completely off-topic or fails to address the question.

Respond with EXACTLY a single floating-point number representing the score. Do not write any other text or reasoning.
"""
    result = call_gemini_judge(prompt)
    try:
        return float(result.strip())
    except ValueError:
        for token in result.split():
            try:
                return float(token)
            except ValueError:
                continue
        return 0.5

def evaluate_context_recall(expected_topics: List[str], retrieved_contexts: List[str]) -> float:
    """Evaluates Context Recall: Did retrieval capture the key doctrinal topics expected?"""
    context_merged = " ".join(retrieved_contexts).lower()
    hits = 0
    for topic in expected_topics:
        if topic.lower() in context_merged:
            hits += 1
    return hits / len(expected_topics) if expected_topics else 1.0

# ==============================================================================
# MAIN RUNNER
# ==============================================================================
def main():
    print("=" * 80)
    print("🚀 STARTING AUTOMATED RAG EVALUATION SUITE FOR THE 7TH HANDLE")
    print(f"Dataset Size: {len(GOLDEN_DATASET)} golden doctrinal test queries")
    print("=" * 80)
    
    results = []
    
    for idx, item in enumerate(GOLDEN_DATASET, 1):
        query = item["query"]
        expected_topics = item["expected_topics"]
        
        print(f"\n[{idx}/{len(GOLDEN_DATASET)}] Evaluating Query: '{query}'...")
        
        # 1. Run RAG Pipeline
        start_time = time.perf_counter()
        response = answer_question(query, answer_mode="Strict Mode")
        latency = time.perf_counter() - start_time
        
        answer = response.get("answer", "")
        sources = response.get("source_documents", [])
        contexts = [doc.page_content for doc in sources]
        
        # 2. Run Evaluators
        faithfulness = evaluate_faithfulness(answer, contexts)
        relevance = evaluate_answer_relevance(query, answer)
        recall = evaluate_context_recall(expected_topics, contexts)
        
        print(f"   -> Latency: {latency:.2f}s | Source Count: {len(sources)}")
        print(f"   -> Faithfulness (Groundedness): {faithfulness:.2f}")
        print(f"   -> Answer Relevance: {relevance:.2f}")
        print(f"   -> Context Recall: {recall:.2f}")
        
        results.append({
            "query": query,
            "latency": latency,
            "faithfulness": faithfulness,
            "relevance": relevance,
            "recall": recall
        })
        
        # Rate limit safety sleep
        time.sleep(1.0)
        
    # ==============================================================================
    # RENDER REPORT TABLE
    # ==============================================================================
    print("\n" + "=" * 80)
    print("📊 FINAL RAG EVALUATION METRICS REPORT")
    print("=" * 80)
    print(f"{'Query':<45} | {'Latency':<7} | {'Faithful':<8} | {'Relevance':<9} | {'Recall':<6}")
    print("-" * 80)
    
    avg_latency = 0.0
    avg_faithfulness = 0.0
    avg_relevance = 0.0
    avg_recall = 0.0
    
    for res in results:
        q_short = res["query"] if len(res["query"]) <= 45 else res["query"][:42] + "..."
        print(f"{q_short:<45} | {res['latency']:5.2f}s | {res['faithfulness']:8.2f} | {res['relevance']:9.2f} | {res['recall']:6.2f}")
        avg_latency += res["latency"]
        avg_faithfulness += res["faithfulness"]
        avg_relevance += res["relevance"]
        avg_recall += res["recall"]
        
    n = len(results)
    print("-" * 80)
    print(f"{'OVERALL AVERAGE':<45} | {avg_latency/n:5.2f}s | {avg_faithfulness/n:8.2f} | {avg_relevance/n:9.2f} | {avg_recall/n:6.2f}")
    print("=" * 80)
    print("🎉 Evaluation suite execution completed successfully!")

if __name__ == "__main__":
    main()