| |
| """ |
| Generate HTML report comparing RAG modes |
| Visualizes benchmark results with charts and tables |
| """ |
|
|
| import json |
| from pathlib import Path |
| from datetime import datetime |
| import statistics |
|
|
|
|
| class RAGComparisonReporter: |
| """Generate HTML reports from benchmark data""" |
|
|
| def __init__(self, results_file: str = "data/benchmark_results.json"): |
| """Initialize reporter with benchmark results""" |
| self.results_file = Path(results_file) |
| self.data = None |
| self.load_results() |
|
|
| def load_results(self): |
| """Load benchmark results from JSON""" |
| if self.results_file.exists(): |
| with open(self.results_file, "r") as f: |
| self.data = json.load(f) |
| else: |
| print(f"Results file not found: {self.results_file}") |
| self.data = {} |
|
|
| def generate_html(self, output_file: str = "rag_comparison_report.html"): |
| """Generate complete HTML report""" |
|
|
| html_content = f""" |
| <!DOCTYPE html> |
| <html lang="en"> |
| <head> |
| <meta charset="UTF-8"> |
| <meta name="viewport" content="width=device-width, initial-scale=1.0"> |
| <title>RAG Comparison Report</title> |
| <script src="https://cdn.jsdelivr.net/npm/chart.js@4.4.0/dist/chart.umd.js"></script> |
| <style> |
| * {{ |
| margin: 0; |
| padding: 0; |
| box-sizing: border-box; |
| }} |
| |
| body {{ |
| font-family: -apple-system, BlinkMacSystemFont, "Segoe UI", Roboto, sans-serif; |
| background: linear-gradient(135deg, #667eea 0%, #764ba2 100%); |
| min-height: 100vh; |
| padding: 40px 20px; |
| }} |
| |
| .container {{ |
| max-width: 1400px; |
| margin: 0 auto; |
| }} |
| |
| .header {{ |
| background: white; |
| border-radius: 12px; |
| padding: 40px; |
| margin-bottom: 30px; |
| box-shadow: 0 10px 30px rgba(0,0,0,0.2); |
| text-align: center; |
| }} |
| |
| .header h1 {{ |
| color: #2d3e50; |
| font-size: 2.5rem; |
| margin-bottom: 10px; |
| }} |
| |
| .header p {{ |
| color: #7f8c8d; |
| font-size: 1rem; |
| }} |
| |
| .timestamp {{ |
| color: #95a5a6; |
| font-size: 0.9rem; |
| margin-top: 10px; |
| }} |
| |
| .grid {{ |
| display: grid; |
| grid-template-columns: repeat(auto-fit, minmax(350px, 1fr)); |
| gap: 30px; |
| margin-bottom: 40px; |
| }} |
| |
| .card {{ |
| background: white; |
| border-radius: 12px; |
| padding: 30px; |
| box-shadow: 0 10px 30px rgba(0,0,0,0.2); |
| }} |
| |
| .card h2 {{ |
| color: #2d3e50; |
| margin-bottom: 20px; |
| font-size: 1.5rem; |
| border-bottom: 2px solid #667eea; |
| padding-bottom: 15px; |
| }} |
| |
| .metric {{ |
| display: flex; |
| justify-content: space-between; |
| align-items: center; |
| padding: 15px 0; |
| border-bottom: 1px solid #ecf0f1; |
| }} |
| |
| .metric:last-child {{ |
| border-bottom: none; |
| }} |
| |
| .metric-label {{ |
| color: #7f8c8d; |
| font-weight: 500; |
| }} |
| |
| .metric-value {{ |
| color: #2d3e50; |
| font-weight: bold; |
| font-size: 1.1rem; |
| }} |
| |
| .badge {{ |
| display: inline-block; |
| background: #667eea; |
| color: white; |
| padding: 8px 12px; |
| border-radius: 6px; |
| font-size: 0.85rem; |
| font-weight: bold; |
| margin: 5px 2px; |
| }} |
| |
| .badge.simple {{ background: #3498db; }} |
| .badge.agentic {{ background: #e74c3c; }} |
| .badge.graph {{ background: #2ecc71; }} |
| |
| .chart-container {{ |
| position: relative; |
| height: 300px; |
| margin: 20px 0; |
| }} |
| |
| table {{ |
| width: 100%; |
| border-collapse: collapse; |
| margin-top: 20px; |
| }} |
| |
| th {{ |
| background: #667eea; |
| color: white; |
| padding: 12px; |
| text-align: left; |
| font-weight: 600; |
| }} |
| |
| td {{ |
| padding: 12px; |
| border-bottom: 1px solid #ecf0f1; |
| }} |
| |
| tr:hover {{ |
| background: #f8f9fa; |
| }} |
| |
| .comparison {{ |
| background: white; |
| border-radius: 12px; |
| padding: 30px; |
| box-shadow: 0 10px 30px rgba(0,0,0,0.2); |
| margin-bottom: 30px; |
| }} |
| |
| .comparison h2 {{ |
| color: #2d3e50; |
| margin-bottom: 20px; |
| font-size: 1.5rem; |
| border-bottom: 2px solid #667eea; |
| padding-bottom: 15px; |
| }} |
| |
| .winner {{ |
| background: #d4edda; |
| border-left: 4px solid #28a745; |
| padding: 15px; |
| margin: 10px 0; |
| border-radius: 6px; |
| }} |
| |
| .winner strong {{ |
| color: #155724; |
| }} |
| |
| .footer {{ |
| background: white; |
| border-radius: 12px; |
| padding: 20px; |
| text-align: center; |
| color: #7f8c8d; |
| margin-top: 40px; |
| box-shadow: 0 10px 30px rgba(0,0,0,0.2); |
| }} |
| |
| @media (max-width: 768px) {{ |
| .grid {{ |
| grid-template-columns: 1fr; |
| }} |
| |
| .header h1 {{ |
| font-size: 1.8rem; |
| }} |
| }} |
| |
| .section {{ |
| margin-bottom: 50px; |
| }} |
| |
| .section-title {{ |
| color: white; |
| font-size: 1.8rem; |
| margin-bottom: 20px; |
| font-weight: bold; |
| }} |
| </style> |
| </head> |
| <body> |
| <div class="container"> |
| {self._generate_header()} |
| {self._generate_executive_summary()} |
| {self._generate_detailed_results()} |
| {self._generate_comparisons()} |
| {self._generate_recommendations()} |
| {self._generate_footer()} |
| </div> |
| |
| <script> |
| {self._generate_charts_script()} |
| </script> |
| </body> |
| </html> |
| """ |
|
|
| output_path = Path(output_file) |
| with open(output_path, "w") as f: |
| f.write(html_content) |
|
|
| print(f"✓ Report generated: {output_path}") |
| return output_path |
|
|
| def _generate_header(self) -> str: |
| """Generate report header""" |
| timestamp = datetime.now().strftime("%Y-%m-%d %H:%M:%S") |
| return f""" |
| <div class="header"> |
| <h1>🔬 RAG Comparison Report</h1> |
| <p>Comprehensive benchmark of Simple, Agentic, and Graph RAG modes</p> |
| <div class="timestamp">Generated: {timestamp}</div> |
| </div> |
| """ |
|
|
| def _generate_executive_summary(self) -> str: |
| """Generate executive summary""" |
| if not self.data.get("benchmarks"): |
| return "" |
|
|
| summary = "<div class='section'><div class='section-title'>📊 Executive Summary</div>" |
| summary += "<div class='grid'>" |
|
|
| for model, results in self.data["benchmarks"].items(): |
| modes = results.get("rag_modes", {}) |
|
|
| summary += """ |
| <div class="card"> |
| <h2>Quick Metrics</h2> |
| """ |
|
|
| |
| if modes: |
| latencies = [ |
| (mode, m["latency"]["mean_ms"]) for mode, m in modes.items() |
| ] |
| latencies.sort(key=lambda x: x[1]) |
|
|
| summary += "<div class='metric'>" |
| summary += "<span class='metric-label'>⚡ Fastest</span>" |
| summary += f"<span class='badge {latencies[0][0]}'>{latencies[0][0].upper()}</span>" |
| summary += f"<span class='metric-value'>{latencies[0][1]:.0f}ms</span>" |
| summary += "</div>" |
|
|
| |
| costs = [ |
| (mode, m["cost_per_query_usd"]) for mode, m in modes.items() |
| ] |
| costs.sort(key=lambda x: x[1]) |
|
|
| summary += "<div class='metric'>" |
| summary += "<span class='metric-label'>💰 Cheapest</span>" |
| summary += f"<span class='badge {costs[0][0]}'>{costs[0][0].upper()}</span>" |
| summary += f"<span class='metric-value'>${costs[0][1]:.4f}</span>" |
| summary += "</div>" |
|
|
| |
| sources = [ |
| (mode, m["sources_avg"]) for mode, m in modes.items() |
| ] |
| sources.sort(key=lambda x: x[1], reverse=True) |
|
|
| summary += "<div class='metric'>" |
| summary += "<span class='metric-label'>📚 Most Sources</span>" |
| summary += f"<span class='badge {sources[0][0]}'>{sources[0][0].upper()}</span>" |
| summary += f"<span class='metric-value'>{sources[0][1]:.1f}</span>" |
| summary += "</div>" |
|
|
| summary += "</div>" |
|
|
| summary += "</div></div>" |
| return summary |
|
|
| def _generate_detailed_results(self) -> str: |
| """Generate detailed results tables""" |
| if not self.data.get("benchmarks"): |
| return "" |
|
|
| html = "<div class='section'><div class='section-title'>📈 Detailed Results</div>" |
|
|
| for model, results in self.data["benchmarks"].items(): |
| modes = results.get("rag_modes", {}) |
|
|
| html += """ |
| <div class="card"> |
| <h2>Model: {}</h2> |
| <table> |
| <tr> |
| <th>RAG Mode</th> |
| <th>Latency (ms)</th> |
| <th>Tokens/Query</th> |
| <th>Sources</th> |
| <th>Cost/Query</th> |
| </tr> |
| """.format(model) |
|
|
| for mode, data in modes.items(): |
| html += f""" |
| <tr> |
| <td><span class="badge {mode}">{mode.upper()}</span></td> |
| <td>{data['latency']['mean_ms']:.0f}</td> |
| <td>{data['tokens']['total_avg']:.0f}</td> |
| <td>{data['sources_avg']:.1f}</td> |
| <td>${data['cost_per_query_usd']:.4f}</td> |
| </tr> |
| """ |
|
|
| html += """ |
| </table> |
| </div> |
| """ |
|
|
| html += "</div>" |
| return html |
|
|
| def _generate_comparisons(self) -> str: |
| """Generate comparison analysis""" |
| if not self.data.get("benchmarks"): |
| return "" |
|
|
| html = "<div class='section'><div class='section-title'>🏆 Comparisons</div>" |
|
|
| for model, results in self.data["benchmarks"].items(): |
| comparisons = results.get("comparisons", {}) |
|
|
| html += """ |
| <div class="comparison"> |
| <h2>Head-to-Head Comparison</h2> |
| """ |
|
|
| if "fastest" in comparisons: |
| fastest = comparisons["fastest"] |
| html += f""" |
| <div class="winner"> |
| <strong>⚡ Fastest Response:</strong> |
| <span class="badge {fastest['mode']}">{fastest['mode'].upper()}</span> |
| - {fastest['latency_ms']:.0f}ms average |
| </div> |
| """ |
|
|
| if "cheapest" in comparisons: |
| cheapest = comparisons["cheapest"] |
| html += f""" |
| <div class="winner"> |
| <strong>💰 Most Cost-Effective:</strong> |
| <span class="badge {cheapest['mode']}">{cheapest['mode'].upper()}</span> |
| - ${cheapest['cost_usd']:.4f} per query |
| </div> |
| """ |
|
|
| if "most_comprehensive" in comparisons: |
| comprehensive = comparisons["most_comprehensive"] |
| html += f""" |
| <div class="winner"> |
| <strong>📚 Most Comprehensive:</strong> |
| <span class="badge {comprehensive['mode']}">{comprehensive['mode'].upper()}</span> |
| - {comprehensive['sources_avg']:.1f} sources average |
| </div> |
| """ |
|
|
| html += """ |
| </div> |
| """ |
|
|
| html += "</div>" |
| return html |
|
|
| def _generate_recommendations(self) -> str: |
| """Generate recommendations""" |
| return """ |
| <div class="section"><div class="section-title">💡 Recommendations</div> |
| <div class="comparison"> |
| <h2>When to Use Each Mode</h2> |
| |
| <div class="winner" style="border-left-color: #3498db;"> |
| <strong>Simple RAG</strong> - Best for: |
| <ul style="margin-left: 20px; margin-top: 10px;"> |
| <li>Real-time applications with <1s latency requirement</li> |
| <li>Direct fact lookup and Q&A</li> |
| <li>Cost-sensitive deployments</li> |
| <li>High-throughput scenarios (>1000 qps)</li> |
| </ul> |
| </div> |
| |
| <div class="winner" style="border-left-color: #e74c3c;"> |
| <strong>Agentic RAG</strong> - Best for: |
| <ul style="margin-left: 20px; margin-top: 10px;"> |
| <li>Complex multi-step reasoning</li> |
| <li>Questions requiring tool use and sub-queries</li> |
| <li>Scenarios where accuracy is critical (>90%)</li> |
| <li>Domain-specific expert systems</li> |
| </ul> |
| </div> |
| |
| <div class="winner" style="border-left-color: #2ecc71;"> |
| <strong>Graph RAG</strong> - Best for: |
| <ul style="margin-left: 20px; margin-top: 10px;"> |
| <li>Knowledge extraction from complex documents</li> |
| <li>Entity and relationship-based queries</li> |
| <li>Balanced latency vs. accuracy (1-2s response)</li> |
| <li>Knowledge bases and expert systems</li> |
| </ul> |
| </div> |
| </div></div> |
| """ |
|
|
| def _generate_footer(self) -> str: |
| """Generate footer""" |
| return """ |
| <div class="footer"> |
| <p>Generated by RAG Comparison Reporter | Data-driven RAG mode selection</p> |
| <p style="margin-top: 10px; font-size: 0.9rem;"> |
| Simple RAG • Agentic RAG • Graph RAG |
| </p> |
| </div> |
| """ |
|
|
| def _generate_charts_script(self) -> str: |
| """Generate Chart.js scripts for visualizations""" |
| if not self.data.get("benchmarks"): |
| return "" |
|
|
| |
| script = """ |
| // Charts would be generated here |
| // Currently showing static data in tables above |
| console.log('RAG Comparison Report loaded'); |
| """ |
|
|
| return script |
|
|
|
|
| def main(): |
| """Generate report from benchmark results""" |
| import argparse |
|
|
| parser = argparse.ArgumentParser(description="Generate RAG comparison HTML report") |
| parser.add_argument( |
| "--input", |
| default="data/benchmark_results.json", |
| help="Input benchmark results file", |
| ) |
| parser.add_argument( |
| "--output", |
| default="rag_comparison_report.html", |
| help="Output HTML file", |
| ) |
|
|
| args = parser.parse_args() |
|
|
| try: |
| reporter = RAGComparisonReporter(args.input) |
| output_file = reporter.generate_html(args.output) |
| print(f"✅ Report generated successfully: {output_file}") |
| print(f" Open in browser: open {output_file}") |
|
|
| except Exception as e: |
| print(f"❌ Error generating report: {e}") |
| import traceback |
| traceback.print_exc() |
| return 1 |
|
|
| return 0 |
|
|
|
|
| if __name__ == "__main__": |
| exit(main()) |
|
|
