import json
import os

PROGRESS_FILES = [
    "data/mintoak/eval_safety_progress.json",
    "data/mintoak/eval_injection_progress.json",
    "data/mintoak/eval_products_progress.json",
    "data/mintoak/eval_greeting_progress.json",
    "data/mintoak/eval_outofscope_progress.json",
    "data/mintoak/eval_general_progress.json",
]
COMBINED_PROGRESS_PATH = "data/mintoak/eval_combined_progress.json"
COMBINED_REPORT_PATH = "outputs/evaluation_report.md"

def main():
    combined_results = {}
    
    # Load all progress files
    for filepath in PROGRESS_FILES:
        if not os.path.exists(filepath):
            print(f"Warning: {filepath} not found. Skipping.")
            continue
            
        print(f"Loading results from {filepath}...")
        try:
            with open(filepath, "r") as f:
                state = json.load(f)
                results_list = state.get("results", [])
                for r in results_list:
                    # Deduplicate by ID to ensure we don't have overlapping runs
                    combined_results[r["id"]] = r
        except Exception as e:
            print(f"Error reading {filepath}: {e}")

    if not combined_results:
        print("No evaluation results found to combine.")
        return

    # Convert back to sorted list of results
    sorted_results = sorted(combined_results.values(), key=lambda x: x["id"])
    
    # Calculate merged metrics
    total_cases = len(sorted_results)
    passed_count = sum(1 for r in sorted_results if r["pass"])
    failed_count = total_cases - passed_count
    pass_rate = (passed_count / total_cases) * 100 if total_cases > 0 else 0.0
    
    total_latency = sum(r["latency"] for r in sorted_results)
    llm_runs = sum(1 for r in sorted_results if r["latency"] > 0)
    avg_latency = (total_latency / llm_runs) if llm_runs > 0 else 0.0
    
    print(f"\nMerged Metrics:")
    print(f"Total processed: {total_cases} cases")
    print(f"Passed: {passed_count} cases")
    print(f"Failed: {failed_count} cases")
    print(f"Pass Rate: {pass_rate:.1f}%")
    print(f"Avg Latency: {avg_latency:.2f}s")
    
    # Save combined progress JSON
    try:
        with open(COMBINED_PROGRESS_PATH, "w") as pf:
            json.dump({
                "results": sorted_results,
                "passed_count": passed_count,
                "total_latency": total_latency,
                "llm_runs": llm_runs
            }, pf, indent=2)
        print(f"Saved merged progress state to '{COMBINED_PROGRESS_PATH}'.")
    except Exception as e:
        print(f"Error saving combined progress state: {e}")

    # Generate merged markdown report
    os.makedirs("outputs", exist_ok=True)
    report_content = f"""# Consolidated End-to-End Evaluation Report

This report consolidates the results from the general RAG pipeline run, the targeted safety profanity runs, and prompt injection runs.

## Summary Metrics

| Metric | Value |
| :--- | :--- |
| **Total Processed Cases** | {total_cases} |
| **Passed Cases** | {passed_count} |
| **Failed Cases** | {failed_count} |
| **Overall Pass Rate** | {pass_rate:.1f}% |
| **Average LLM Inference Latency** | {avg_latency:.2f}s |

## Test Case Results

"""
    for r in sorted_results:
        status_emoji = "✅ PASS" if r["pass"] else "❌ FAIL"
        report_content += f"""### [{r['id']}] {r['category'].upper()} - {status_emoji}
* **Query**: `{r['query']}`
* **Latency**: {r['latency']:.2f}s
* **Response**:
```text
{r['response']}
```
"""
        if not r["pass"]:
            report_content += f"* **Failure Reason**: {r['reason']}\n"
        report_content += "\n---\n\n"

    try:
        with open(COMBINED_REPORT_PATH, "w") as rf:
            rf.write(report_content)
        print(f"Saved consolidated markdown report to '{COMBINED_REPORT_PATH}'.")
    except Exception as e:
        print(f"Error writing markdown report: {e}")

if __name__ == "__main__":
    main()