Spaces:
Paused
Paused
| import json | |
| import os | |
| PROGRESS_FILES = [ | |
| "data/mintoak/eval_safety_progress.json", | |
| "data/mintoak/eval_injection_progress.json", | |
| "data/mintoak/eval_products_progress.json", | |
| "data/mintoak/eval_greeting_progress.json", | |
| "data/mintoak/eval_outofscope_progress.json", | |
| "data/mintoak/eval_general_progress.json", | |
| ] | |
| COMBINED_PROGRESS_PATH = "data/mintoak/eval_combined_progress.json" | |
| COMBINED_REPORT_PATH = "outputs/evaluation_report.md" | |
| def main(): | |
| combined_results = {} | |
| # Load all progress files | |
| for filepath in PROGRESS_FILES: | |
| if not os.path.exists(filepath): | |
| print(f"Warning: {filepath} not found. Skipping.") | |
| continue | |
| print(f"Loading results from {filepath}...") | |
| try: | |
| with open(filepath, "r") as f: | |
| state = json.load(f) | |
| results_list = state.get("results", []) | |
| for r in results_list: | |
| # Deduplicate by ID to ensure we don't have overlapping runs | |
| combined_results[r["id"]] = r | |
| except Exception as e: | |
| print(f"Error reading {filepath}: {e}") | |
| if not combined_results: | |
| print("No evaluation results found to combine.") | |
| return | |
| # Convert back to sorted list of results | |
| sorted_results = sorted(combined_results.values(), key=lambda x: x["id"]) | |
| # Calculate merged metrics | |
| total_cases = len(sorted_results) | |
| passed_count = sum(1 for r in sorted_results if r["pass"]) | |
| failed_count = total_cases - passed_count | |
| pass_rate = (passed_count / total_cases) * 100 if total_cases > 0 else 0.0 | |
| total_latency = sum(r["latency"] for r in sorted_results) | |
| llm_runs = sum(1 for r in sorted_results if r["latency"] > 0) | |
| avg_latency = (total_latency / llm_runs) if llm_runs > 0 else 0.0 | |
| print(f"\nMerged Metrics:") | |
| print(f"Total processed: {total_cases} cases") | |
| print(f"Passed: {passed_count} cases") | |
| print(f"Failed: {failed_count} cases") | |
| print(f"Pass Rate: {pass_rate:.1f}%") | |
| print(f"Avg Latency: {avg_latency:.2f}s") | |
| # Save combined progress JSON | |
| try: | |
| with open(COMBINED_PROGRESS_PATH, "w") as pf: | |
| json.dump({ | |
| "results": sorted_results, | |
| "passed_count": passed_count, | |
| "total_latency": total_latency, | |
| "llm_runs": llm_runs | |
| }, pf, indent=2) | |
| print(f"Saved merged progress state to '{COMBINED_PROGRESS_PATH}'.") | |
| except Exception as e: | |
| print(f"Error saving combined progress state: {e}") | |
| # Generate merged markdown report | |
| os.makedirs("outputs", exist_ok=True) | |
| report_content = f"""# Consolidated End-to-End Evaluation Report | |
| This report consolidates the results from the general RAG pipeline run, the targeted safety profanity runs, and prompt injection runs. | |
| ## Summary Metrics | |
| | Metric | Value | | |
| | :--- | :--- | | |
| | **Total Processed Cases** | {total_cases} | | |
| | **Passed Cases** | {passed_count} | | |
| | **Failed Cases** | {failed_count} | | |
| | **Overall Pass Rate** | {pass_rate:.1f}% | | |
| | **Average LLM Inference Latency** | {avg_latency:.2f}s | | |
| ## Test Case Results | |
| """ | |
| for r in sorted_results: | |
| status_emoji = "✅ PASS" if r["pass"] else "❌ FAIL" | |
| report_content += f"""### [{r['id']}] {r['category'].upper()} - {status_emoji} | |
| * **Query**: `{r['query']}` | |
| * **Latency**: {r['latency']:.2f}s | |
| * **Response**: | |
| ```text | |
| {r['response']} | |
| ``` | |
| """ | |
| if not r["pass"]: | |
| report_content += f"* **Failure Reason**: {r['reason']}\n" | |
| report_content += "\n---\n\n" | |
| try: | |
| with open(COMBINED_REPORT_PATH, "w") as rf: | |
| rf.write(report_content) | |
| print(f"Saved consolidated markdown report to '{COMBINED_REPORT_PATH}'.") | |
| except Exception as e: | |
| print(f"Error writing markdown report: {e}") | |
| if __name__ == "__main__": | |
| main() | |