#!/usr/bin/env python3 """ Generate HTML report comparing RAG modes Visualizes benchmark results with charts and tables """ import json from pathlib import Path from datetime import datetime import statistics class RAGComparisonReporter: """Generate HTML reports from benchmark data""" def __init__(self, results_file: str = "data/benchmark_results.json"): """Initialize reporter with benchmark results""" self.results_file = Path(results_file) self.data = None self.load_results() def load_results(self): """Load benchmark results from JSON""" if self.results_file.exists(): with open(self.results_file, "r") as f: self.data = json.load(f) else: print(f"Results file not found: {self.results_file}") self.data = {} def generate_html(self, output_file: str = "rag_comparison_report.html"): """Generate complete HTML report""" html_content = f""" RAG Comparison Report
{self._generate_header()} {self._generate_executive_summary()} {self._generate_detailed_results()} {self._generate_comparisons()} {self._generate_recommendations()} {self._generate_footer()}
""" output_path = Path(output_file) with open(output_path, "w") as f: f.write(html_content) print(f"✓ Report generated: {output_path}") return output_path def _generate_header(self) -> str: """Generate report header""" timestamp = datetime.now().strftime("%Y-%m-%d %H:%M:%S") return f"""

🔬 RAG Comparison Report

Comprehensive benchmark of Simple, Agentic, and Graph RAG modes

Generated: {timestamp}
""" def _generate_executive_summary(self) -> str: """Generate executive summary""" if not self.data.get("benchmarks"): return "" summary = "
📊 Executive Summary
" summary += "
" for model, results in self.data["benchmarks"].items(): modes = results.get("rag_modes", {}) summary += """

Quick Metrics

""" # Latency comparison if modes: latencies = [ (mode, m["latency"]["mean_ms"]) for mode, m in modes.items() ] latencies.sort(key=lambda x: x[1]) summary += "
" summary += "⚡ Fastest" summary += f"{latencies[0][0].upper()}" summary += f"{latencies[0][1]:.0f}ms" summary += "
" # Cost comparison costs = [ (mode, m["cost_per_query_usd"]) for mode, m in modes.items() ] costs.sort(key=lambda x: x[1]) summary += "
" summary += "💰 Cheapest" summary += f"{costs[0][0].upper()}" summary += f"${costs[0][1]:.4f}" summary += "
" # Most comprehensive sources = [ (mode, m["sources_avg"]) for mode, m in modes.items() ] sources.sort(key=lambda x: x[1], reverse=True) summary += "
" summary += "📚 Most Sources" summary += f"{sources[0][0].upper()}" summary += f"{sources[0][1]:.1f}" summary += "
" summary += "
" summary += "
" return summary def _generate_detailed_results(self) -> str: """Generate detailed results tables""" if not self.data.get("benchmarks"): return "" html = "
📈 Detailed Results
" for model, results in self.data["benchmarks"].items(): modes = results.get("rag_modes", {}) html += """

Model: {}

""".format(model) for mode, data in modes.items(): html += f""" """ html += """
RAG Mode Latency (ms) Tokens/Query Sources Cost/Query
{mode.upper()} {data['latency']['mean_ms']:.0f} {data['tokens']['total_avg']:.0f} {data['sources_avg']:.1f} ${data['cost_per_query_usd']:.4f}
""" html += "
" return html def _generate_comparisons(self) -> str: """Generate comparison analysis""" if not self.data.get("benchmarks"): return "" html = "
🏆 Comparisons
" for model, results in self.data["benchmarks"].items(): comparisons = results.get("comparisons", {}) html += """

Head-to-Head Comparison

""" if "fastest" in comparisons: fastest = comparisons["fastest"] html += f"""
⚡ Fastest Response: {fastest['mode'].upper()} - {fastest['latency_ms']:.0f}ms average
""" if "cheapest" in comparisons: cheapest = comparisons["cheapest"] html += f"""
💰 Most Cost-Effective: {cheapest['mode'].upper()} - ${cheapest['cost_usd']:.4f} per query
""" if "most_comprehensive" in comparisons: comprehensive = comparisons["most_comprehensive"] html += f"""
📚 Most Comprehensive: {comprehensive['mode'].upper()} - {comprehensive['sources_avg']:.1f} sources average
""" html += """
""" html += "
" return html def _generate_recommendations(self) -> str: """Generate recommendations""" return """
💡 Recommendations

When to Use Each Mode

Simple RAG - Best for:
  • Real-time applications with <1s latency requirement
  • Direct fact lookup and Q&A
  • Cost-sensitive deployments
  • High-throughput scenarios (>1000 qps)
Agentic RAG - Best for:
  • Complex multi-step reasoning
  • Questions requiring tool use and sub-queries
  • Scenarios where accuracy is critical (>90%)
  • Domain-specific expert systems
Graph RAG - Best for:
  • Knowledge extraction from complex documents
  • Entity and relationship-based queries
  • Balanced latency vs. accuracy (1-2s response)
  • Knowledge bases and expert systems
""" def _generate_footer(self) -> str: """Generate footer""" return """

Generated by RAG Comparison Reporter | Data-driven RAG mode selection

Simple RAG • Agentic RAG • Graph RAG

""" def _generate_charts_script(self) -> str: """Generate Chart.js scripts for visualizations""" if not self.data.get("benchmarks"): return "" # Extract data for charts script = """ // Charts would be generated here // Currently showing static data in tables above console.log('RAG Comparison Report loaded'); """ return script def main(): """Generate report from benchmark results""" import argparse parser = argparse.ArgumentParser(description="Generate RAG comparison HTML report") parser.add_argument( "--input", default="data/benchmark_results.json", help="Input benchmark results file", ) parser.add_argument( "--output", default="rag_comparison_report.html", help="Output HTML file", ) args = parser.parse_args() try: reporter = RAGComparisonReporter(args.input) output_file = reporter.generate_html(args.output) print(f"✅ Report generated successfully: {output_file}") print(f" Open in browser: open {output_file}") except Exception as e: print(f"❌ Error generating report: {e}") import traceback traceback.print_exc() return 1 return 0 if __name__ == "__main__": exit(main())