Spaces:
Running on Zero
Running on Zero
File size: 2,732 Bytes
6a5d1ee | 1 2 3 4 5 6 7 8 9 10 11 12 13 14 15 16 17 18 19 20 21 22 23 24 25 26 27 28 29 30 31 32 33 34 35 36 37 38 39 40 41 42 43 44 45 46 47 48 49 50 51 52 53 54 55 56 57 58 59 60 61 62 63 64 65 66 67 68 | #!/usr/bin/env python3
"""
Run the real evaluation (Basic Vector RAG vs Hybrid RAG vs Agentic RAG) and
the ablation study, print a summary table, and save full results to
data/eval_results.json. Every number is computed live against the corpus in
data/corpus.json using the labeled questions in data/eval_questions.json.
Usage:
python scripts/evaluate.py
"""
import json
import logging
import os
import sys
sys.path.insert(0, os.path.dirname(os.path.dirname(os.path.abspath(__file__))))
logging.basicConfig(level=logging.ERROR)
from src.agents.retrieval import init_rag, rag_status # noqa: E402
from src.evaluation.benchmark import run_ablation, run_benchmark, summarize # noqa: E402
CORPUS_PATH = os.path.join(os.path.dirname(os.path.dirname(os.path.abspath(__file__))), "data", "corpus.json")
EVAL_PATH = os.path.join(os.path.dirname(os.path.dirname(os.path.abspath(__file__))), "data", "eval_questions.json")
OUT_PATH = os.path.join(os.path.dirname(os.path.dirname(os.path.abspath(__file__))), "data", "eval_results.json")
def _print_table(title: str, summary: dict) -> None:
print(f"\n=== {title} ===")
metric_names = list(next(iter(summary.values())).keys())
header = f"{'System':<34}" + "".join(f"{m:>22}" for m in metric_names)
print(header)
print("-" * len(header))
for system, metrics in summary.items():
row = f"{system:<34}" + "".join(f"{metrics[m]:>22}" for m in metric_names)
print(row)
def main():
init_rag(CORPUS_PATH)
status = rag_status()
print(f"RAG backend in use for this evaluation run: {status['embedding_backend']} "
f"(reranker: {status['reranker_backend']})")
print(f"Corpus: {status['num_documents']} documents / {status['num_chunks']} chunks")
print("\nRunning system comparison (Basic Vector RAG vs Hybrid RAG vs Agentic RAG)...")
comparison_results = run_benchmark(CORPUS_PATH, EVAL_PATH)
comparison_summary = summarize(comparison_results)
_print_table("System Comparison (mean over eval set)", comparison_summary)
print("\nRunning ablation study on the Agentic RAG pipeline...")
ablation_results = run_ablation(CORPUS_PATH, EVAL_PATH)
ablation_summary = summarize(ablation_results)
_print_table("Ablation Study (mean over eval set)", ablation_summary)
with open(OUT_PATH, "w", encoding="utf-8") as f:
json.dump({
"rag_backend": status,
"system_comparison_raw": comparison_results,
"system_comparison_summary": comparison_summary,
"ablation_raw": ablation_results,
"ablation_summary": ablation_summary,
}, f, indent=2)
print(f"\nFull results saved to {OUT_PATH}")
if __name__ == "__main__":
main()
|