scripts/evaluate.py

#!/usr/bin/env python3
"""
Run the real evaluation (Basic Vector RAG vs Hybrid RAG vs Agentic RAG) and
the ablation study, print a summary table, and save full results to
data/eval_results.json. Every number is computed live against the corpus in
data/corpus.json using the labeled questions in data/eval_questions.json.

Usage:
    python scripts/evaluate.py
"""
import json
import logging
import os
import sys

sys.path.insert(0, os.path.dirname(os.path.dirname(os.path.abspath(__file__))))
logging.basicConfig(level=logging.ERROR)

from src.agents.retrieval import init_rag, rag_status  # noqa: E402
from src.evaluation.benchmark import run_ablation, run_benchmark, summarize  # noqa: E402

CORPUS_PATH = os.path.join(os.path.dirname(os.path.dirname(os.path.abspath(__file__))), "data", "corpus.json")
EVAL_PATH = os.path.join(os.path.dirname(os.path.dirname(os.path.abspath(__file__))), "data", "eval_questions.json")
OUT_PATH = os.path.join(os.path.dirname(os.path.dirname(os.path.abspath(__file__))), "data", "eval_results.json")


def _print_table(title: str, summary: dict) -> None:
    print(f"\n=== {title} ===")
    metric_names = list(next(iter(summary.values())).keys())
    header = f"{'System':<34}" + "".join(f"{m:>22}" for m in metric_names)
    print(header)
    print("-" * len(header))
    for system, metrics in summary.items():
        row = f"{system:<34}" + "".join(f"{metrics[m]:>22}" for m in metric_names)
        print(row)


def main():
    init_rag(CORPUS_PATH)
    status = rag_status()
    print(f"RAG backend in use for this evaluation run: {status['embedding_backend']} "
          f"(reranker: {status['reranker_backend']})")
    print(f"Corpus: {status['num_documents']} documents / {status['num_chunks']} chunks")

    print("\nRunning system comparison (Basic Vector RAG vs Hybrid RAG vs Agentic RAG)...")
    comparison_results = run_benchmark(CORPUS_PATH, EVAL_PATH)
    comparison_summary = summarize(comparison_results)
    _print_table("System Comparison (mean over eval set)", comparison_summary)

    print("\nRunning ablation study on the Agentic RAG pipeline...")
    ablation_results = run_ablation(CORPUS_PATH, EVAL_PATH)
    ablation_summary = summarize(ablation_results)
    _print_table("Ablation Study (mean over eval set)", ablation_summary)

    with open(OUT_PATH, "w", encoding="utf-8") as f:
        json.dump({
            "rag_backend": status,
            "system_comparison_raw": comparison_results,
            "system_comparison_summary": comparison_summary,
            "ablation_raw": ablation_results,
            "ablation_summary": ablation_summary,
        }, f, indent=2)
    print(f"\nFull results saved to {OUT_PATH}")


if __name__ == "__main__":
    main()