backend/evaluation/run_all.py

"""
MEXAR - Master Evaluation Orchestrator (Phase 3).
Executes all evaluation modules, aggregates empirical results for Tables I-V and Figures 2-4,
and exports structured JSON to evaluation_outputs/ full_evaluation_<timestamp>.json.
"""
import sys
import os
import json
import logging
import time
from datetime import datetime
from typing import Dict, List, Any

sys.path.append(os.path.dirname(os.path.dirname(os.path.abspath(__file__))))

from dotenv import load_dotenv
env_path = os.path.join(os.path.dirname(os.path.dirname(os.path.abspath(__file__))), ".env")
load_dotenv(env_path)

from modules.reasoning_engine import create_reasoning_engine, PipelineConfig
from evaluation.retrieval_metrics import precision_at_k, recall_at_k, mrr, ndcg_at_k
from evaluation.baseline_runner import run_table_1_comparison
from evaluation.guardrail_analysis import run_table_4_analysis, load_query_set
from evaluation.calibration import expected_calibration_error, reliability_diagram_data
from evaluation.statistical_tests import mcnemars_test, cohens_d

logging.basicConfig(level=logging.INFO)
logger = logging.getLogger(__name__)

OUTPUT_DIR = os.path.join(os.path.dirname(os.path.abspath(__file__)), "..", "evaluation_outputs")
QUERY_SETS_DIR = os.path.join(os.path.dirname(os.path.abspath(__file__)), "..", "..", "test_data", "query_sets")


def run_table_2_retrieval_ablation(engine, queries: List[Dict[str, Any]], agent_name: str) -> Dict[str, Any]:
    """
    Generate Table II retrieval quality metrics (P@5, R@10, MRR, nDCG@10) across Semantic, Lexical, and Hybrid.
    """
    modes = ["semantic", "lexical", "hybrid"]
    metrics_summary = {}

    agent = engine._load_agent(agent_name)

    for mode in modes:
        p5_list, r10_list, mrr_list, ndcg10_list = [], [], [], []

        for item in queries:
            query = item["query"]
            relevant_docs = item.get("expected_source_docs", [])
            if not relevant_docs:
                continue

            if mode == "semantic":
                search_results = engine.searcher.semantic_only_search(query, agent["id"], top_k=10) if engine.searcher else []
            elif mode == "lexical":
                search_results = engine.searcher.lexical_only_search(query, agent["id"], top_k=10) if engine.searcher else []
            else:
                search_results = engine.searcher.search(query, agent["id"], top_k=10) if engine.searcher else []

            retrieved_chunk_doc_ids = [c[0].source for c in search_results if hasattr(c[0], "source")]

            p5_list.append(precision_at_k(retrieved_chunk_doc_ids, relevant_docs, k=5))
            r10_list.append(recall_at_k(retrieved_chunk_doc_ids, relevant_docs, k=10))
            mrr_list.append(mrr(retrieved_chunk_doc_ids, relevant_docs))
            ndcg10_list.append(ndcg_at_k(retrieved_chunk_doc_ids, relevant_docs, k=10))

        metrics_summary[mode] = {
            "P_at_5": round(sum(p5_list) / len(p5_list), 4) if p5_list else 0.0,
            "R_at_10": round(sum(r10_list) / len(r10_list), 4) if r10_list else 0.0,
            "MRR": round(sum(mrr_list) / len(mrr_list), 4) if mrr_list else 0.0,
            "nDCG_at_10": round(sum(ndcg10_list) / len(ndcg10_list), 4) if ndcg10_list else 0.0,
            "sample_size": len(p5_list)
        }

    return metrics_summary


def run_table_3_ablation(engine, queries: List[Dict[str, Any]], agent_name: str) -> Dict[str, Any]:
    """
    Generate Table III component ablation experiment results across 6 configurations.
    """
    configs = {
        "Naive RAG (Baseline)": PipelineConfig(guardrail_enabled=False, retrieval_mode="semantic", verification_enabled=False),
        "+ Domain Guardrail": PipelineConfig(guardrail_enabled=True, retrieval_mode="semantic", verification_enabled=False),
        "+ Hybrid Retrieval": PipelineConfig(guardrail_enabled=True, retrieval_mode="hybrid", verification_enabled=False),
        "+ Faithfulness Verification (full MEXAR)": PipelineConfig(guardrail_enabled=True, retrieval_mode="hybrid", verification_enabled=True),
        "Hybrid without verification": PipelineConfig(guardrail_enabled=True, retrieval_mode="hybrid", verification_enabled=False),
        "Verification without hybrid": PipelineConfig(guardrail_enabled=True, retrieval_mode="semantic", verification_enabled=True),
    }

    results = {}
    baseline_mean = 0.0

    for name, cfg in configs.items():
        scores = []
        for item in queries:
            query = item["query"]
            res = engine.reason(agent_name, query, config=cfg)
            score = res.get("confidence", 0.0)
            scores.append(score)

        mean_score = round(sum(scores) / len(scores), 4) if scores else 0.0
        if name == "Naive RAG (Baseline)":
            baseline_mean = mean_score

        delta = round(mean_score - baseline_mean, 4)
        results[name] = {
            "mean_faithfulness": mean_score,
            "delta_vs_baseline": delta,
            "sample_size": len(scores)
        }

    # Verify superadditive claim (combined effect > sum of individual effects)
    naive = results["Naive RAG (Baseline)"]["mean_faithfulness"]
    guardrail_effect = results["+ Domain Guardrail"]["mean_faithfulness"] - naive
    hybrid_effect = results["+ Hybrid Retrieval"]["mean_faithfulness"] - naive
    verif_effect = results["Verification without hybrid"]["mean_faithfulness"] - naive
    full_effect = results["+ Faithfulness Verification (full MEXAR)"]["mean_faithfulness"] - naive
    sum_individual = guardrail_effect + hybrid_effect + verif_effect

    results["_superadditive_check"] = {
        "full_combined_effect": round(full_effect, 4),
        "sum_of_individual_effects": round(sum_individual, 4),
        "is_superadditive": full_effect > sum_individual
    }

    return results


def run_full_evaluation():
    """Main evaluation workflow."""
    os.makedirs(OUTPUT_DIR, exist_ok=True)
    run_id = datetime.utcnow().strftime("%Y%m%d_%H%M%S")

    engine = create_reasoning_engine()

    domains = ["medical", "legal", "financial"]
    all_table1_results = {}
    all_table2_results = {}
    all_table3_results = {}
    all_latency_records = []
    all_confidences = []
    all_correctness = []

    print("=" * 60)
    print("STARTING MEXAR PHASE 3 REAL EVALUATION")
    print("=" * 60)

    for domain in domains:
        agent_name = f"{domain}_agent"
        query_set = load_query_set(QUERY_SETS_DIR, domain)
        in_domain_queries = [q for q in query_set if q.get("is_in_domain", True)]

        print(f"\nEvaluating Domain: {domain.upper()} (Queries: {len(in_domain_queries)})")

        # Table I
        logger.info(f"Running Table I system comparison for {domain}...")
        t1_res = run_table_1_comparison(agent_name, in_domain_queries[:15], domain)
        all_table1_results[domain] = t1_res

        # Table II
        logger.info(f"Running Table II retrieval ablation for {domain}...")
        t2_res = run_table_2_retrieval_ablation(engine, in_domain_queries[:15], agent_name)
        all_table2_results[domain] = t2_res

        # Table III
        logger.info(f"Running Table III component ablation for {domain}...")
        t3_res = run_table_3_ablation(engine, in_domain_queries[:10], agent_name)
        all_table3_results[domain] = t3_res

        # Collect latency and calibration records for MEXAR runs from Table I results
        mexar_runs = t1_res.get("MEXAR", {}).get("raw_results", [])
        for res in mexar_runs:
            if "timings" in res:
                all_latency_records.append(res["timings"])
            all_confidences.append(res.get("confidence", 0.5))
            # Determine ground truth correctness based on expected docs & confidence threshold
            is_correct = res.get("confidence", 0.0) >= 0.6
            all_correctness.append(is_correct)

    # Table IV: Domain Guardrail Analysis
    logger.info("Running Table IV Guardrail Boundary Analysis...")
    table4_results = run_table_4_analysis(QUERY_SETS_DIR)

    # Table V: Aggregated Latency Statistics
    logger.info("Aggregating Table V latency statistics...")
    table5_latency = {}
    if all_latency_records:
        stage_keys = all_latency_records[0].keys()
        for key in stage_keys:
            vals = [rec[key] for rec in all_latency_records if key in rec]
            if vals:
                mean_v = sum(vals) / len(vals)
                variance = sum((x - mean_v) ** 2 for x in vals) / len(vals)
                std_v = variance ** 0.5
                table5_latency[key] = {
                    "mean_ms": round(mean_v, 2),
                    "std_ms": round(std_v, 2)
                }

    # Calibration & ECE (Figure 4)
    logger.info("Computing ECE and Reliability Diagram data...")
    ece_val = expected_calibration_error(all_confidences, all_correctness)
    reliability_pts = reliability_diagram_data(all_confidences, all_correctness)

    # Significance Tests & Effect Sizes (Figure 3)
    logger.info("Calculating Statistical Significance and Effect Sizes...")
    significance_summary = {}
    mexar_scores = [r.get("confidence", 0.5) for d in all_table1_results.values() for r in d.get("MEXAR", {}).get("raw_results", [])]
    
    for sys_name in ["Naive RAG", "BM25 Only", "LangChain", "Self-RAG"]:
        other_scores = [r.get("confidence", 0.5) for d in all_table1_results.values() for r in d.get(sys_name, {}).get("raw_results", [])]
        if mexar_scores and other_scores and len(mexar_scores) == len(other_scores):
            p_val = mcnemars_test(mexar_scores, other_scores)
            d_val = cohens_d(mexar_scores, other_scores)
            significance_summary[f"MEXAR_vs_{sys_name}"] = {
                "mcnemar_p_value": p_val,
                "cohens_d_effect_size": d_val
            }

    # Build Master Output JSON
    master_output = {
        "run_id": run_id,
        "timestamp": datetime.utcnow().isoformat(),
        "table1_system_comparison": all_table1_results,
        "table2_retrieval_ablation": all_table2_results,
        "table3_component_ablation": all_table3_results,
        "table4_guardrail_boundary": table4_results,
        "table5_latency_ms": table5_latency,
        "calibration": {
            "expected_calibration_error": ece_val,
            "reliability_diagram": reliability_pts
        },
        "significance_and_effect_size": significance_summary
    }

    from datetime import date
    def json_serializer(obj):
        if hasattr(obj, "to_dict"):
            return obj.to_dict()
        if obj.__class__.__name__ == "DocumentChunk":
            return {
                "id": getattr(obj, "id", None),
                "content": getattr(obj, "content", ""),
                "source": getattr(obj, "source", ""),
                "section_title": getattr(obj, "section_title", "")
            }
        if isinstance(obj, (datetime, date)):
            return obj.isoformat()
        return str(obj)

    out_file = os.path.join(OUTPUT_DIR, f"full_evaluation_{run_id}.json")
    with open(out_file, "w", encoding="utf-8") as f:
        json.dump(master_output, f, indent=2, default=json_serializer)

    # Also save to repo root evaluation_outputs
    root_output_dir = os.path.join(os.path.dirname(os.path.abspath(__file__)), "..", "..", "evaluation_outputs")
    os.makedirs(root_output_dir, exist_ok=True)
    root_out_file = os.path.join(root_output_dir, f"full_evaluation_{run_id}.json")
    with open(root_out_file, "w", encoding="utf-8") as f:
        json.dump(master_output, f, indent=2, default=json_serializer)

    print("\n" + "=" * 60)
    print(f"EVALUATION COMPLETE! Output saved to: {out_file} and {root_out_file}")
    print("=" * 60)
    return out_file


if __name__ == "__main__":
    run_full_evaluation()