import json import os PROGRESS_FILE = "data/mintoak/eval_combined_progress.json" OUTPUT_HTML = "outputs/evaluation_report.html" CATEGORY_LABELS = { "greeting": "Greetings", "general_inquiry": "Inquiries", "out_of_scope": "Out of Scope", "guardrail_safety": "Safety", "guardrail_injection": "Injections", } CATEGORY_COLORS = { "greeting": "#6366f1", "general_inquiry": "#8b5cf6", "out_of_scope": "#f59e0b", "guardrail_safety": "#10b981", "guardrail_injection": "#ef4444", } def main(): if not os.path.exists(PROGRESS_FILE): print(f"Error: Progress file {PROGRESS_FILE} not found. Please run evaluation first.") return print(f"Loading results from {PROGRESS_FILE}...") with open(PROGRESS_FILE, "r") as f: state = json.load(f) results = state.get("results", []) # Always remap product_inquiry → general_inquiry at read-time CATEGORY_REMAP = {"product_inquiry": "general_inquiry"} for r in results: if r["category"] in CATEGORY_REMAP: r["category"] = CATEGORY_REMAP[r["category"]] passed_count = sum(1 for r in results if r["pass"]) total_cases = len(results) failed_count = total_cases - passed_count pass_rate = (passed_count / total_cases) * 100 if total_cases > 0 else 0.0 total_latency = state.get("total_latency", 0.0) llm_runs = state.get("llm_runs", 0) avg_latency = (total_latency / llm_runs) if llm_runs > 0 else 0.0 # Per-category stats cat_stats = {} for r in results: cat = r["category"] if cat not in cat_stats: cat_stats[cat] = {"total": 0, "passed": 0, "latencies": []} cat_stats[cat]["total"] += 1 if r["pass"]: cat_stats[cat]["passed"] += 1 if r["latency"] > 0: cat_stats[cat]["latencies"].append(r["latency"]) cat_order = ["greeting", "general_inquiry", "out_of_scope", "guardrail_safety", "guardrail_injection"] cat_order = [c for c in cat_order if c in cat_stats] + [c for c in cat_stats if c not in cat_order] # Build JS arrays for charts bar_labels = json.dumps([CATEGORY_LABELS.get(c, c) for c in cat_order]) bar_totals = json.dumps([cat_stats[c]["total"] for c in cat_order]) bar_passed = json.dumps([cat_stats[c]["passed"] for c in cat_order]) bar_failed = json.dumps([cat_stats[c]["total"] - cat_stats[c]["passed"] for c in cat_order]) bar_pass_rates = json.dumps([round(cat_stats[c]["passed"] / cat_stats[c]["total"] * 100, 1) for c in cat_order]) bar_colors = json.dumps([CATEGORY_COLORS.get(c, "#94a3b8") for c in cat_order]) avg_latencies = json.dumps([ round(sum(cat_stats[c]["latencies"]) / len(cat_stats[c]["latencies"]), 2) if cat_stats[c]["latencies"] else 0 for c in cat_order ]) # Category tile data for the summary strip cat_tiles_html = "" for cat in cat_order: s = cat_stats[cat] rate = round(s["passed"] / s["total"] * 100, 1) if s["total"] else 0 color = CATEGORY_COLORS.get(cat, "#94a3b8") label = CATEGORY_LABELS.get(cat, cat) cat_tiles_html += f"""
{label} {s['total']} cases
{rate}%
""" # Build case cards HTML cards_html = "" for r in results: status_class = "pass" if r["pass"] else "fail" status_text = "Pass" if r["pass"] else "Fail" cat_color = CATEGORY_COLORS.get(r["category"], "#94a3b8") cat_label = CATEGORY_LABELS.get(r["category"], r["category"]) fail_section = "" if r["pass"] else f"""
⚠ Failure Reason
{r["reason"]}
""" lat_badge = f'{r["latency"]:.2f}s' if r["latency"] > 0 else "instant" cards_html += f"""
{r['id']} {cat_label} {r['query']}
⏱ {lat_badge} {status_text} ›
Query
{r['query']}
{fail_section}
Assistant Response
{r['response']}
""" html = f""" Mintoak RAG Evaluation Dashboard
Mintoak RAG · Evaluation Dashboard
{total_cases} test cases · generated {__import__('datetime').datetime.now().strftime('%d %b %Y, %H:%M')}
✓ {pass_rate:.1f}% Pass Rate
Total Cases
{total_cases}
Across all categories
Passed
{passed_count}
of {total_cases} cases
Failed
{failed_count}
of {total_cases} cases
Pass Rate
{pass_rate:.1f}%
Overall alignment
Avg Latency
{avg_latency:.2f}s
LLM inference only

Analytics Overview

Pass / Fail Ratio
{pass_rate:.1f}%
pass rate
Cases by Category
Avg Latency per Category (s)

Filter by Category

All Categories {total_cases} cases
{pass_rate:.1f}%
{cat_tiles_html}
Showing {total_cases} cases
{cards_html}
""" os.makedirs("outputs", exist_ok=True) with open(OUTPUT_HTML, "w", encoding="utf-8") as f: f.write(html) print(f"Successfully generated HTML report at '{OUTPUT_HTML}'!") if __name__ == "__main__": main()