""" MEXAR - Master Evaluation Orchestrator (Phase 3). Executes all evaluation modules, aggregates empirical results for Tables I-V and Figures 2-4, and exports structured JSON to evaluation_outputs/ full_evaluation_.json. """ import sys import os import json import logging import time from datetime import datetime from typing import Dict, List, Any sys.path.append(os.path.dirname(os.path.dirname(os.path.abspath(__file__)))) from dotenv import load_dotenv env_path = os.path.join(os.path.dirname(os.path.dirname(os.path.abspath(__file__))), ".env") load_dotenv(env_path) from modules.reasoning_engine import create_reasoning_engine, PipelineConfig from evaluation.retrieval_metrics import precision_at_k, recall_at_k, mrr, ndcg_at_k from evaluation.baseline_runner import run_table_1_comparison from evaluation.guardrail_analysis import run_table_4_analysis, load_query_set from evaluation.calibration import expected_calibration_error, reliability_diagram_data from evaluation.statistical_tests import mcnemars_test, cohens_d logging.basicConfig(level=logging.INFO) logger = logging.getLogger(__name__) OUTPUT_DIR = os.path.join(os.path.dirname(os.path.abspath(__file__)), "..", "evaluation_outputs") QUERY_SETS_DIR = os.path.join(os.path.dirname(os.path.abspath(__file__)), "..", "..", "test_data", "query_sets") def run_table_2_retrieval_ablation(engine, queries: List[Dict[str, Any]], agent_name: str) -> Dict[str, Any]: """ Generate Table II retrieval quality metrics (P@5, R@10, MRR, nDCG@10) across Semantic, Lexical, and Hybrid. """ modes = ["semantic", "lexical", "hybrid"] metrics_summary = {} agent = engine._load_agent(agent_name) for mode in modes: p5_list, r10_list, mrr_list, ndcg10_list = [], [], [], [] for item in queries: query = item["query"] relevant_docs = item.get("expected_source_docs", []) if not relevant_docs: continue if mode == "semantic": search_results = engine.searcher.semantic_only_search(query, agent["id"], top_k=10) if engine.searcher else [] elif mode == "lexical": search_results = engine.searcher.lexical_only_search(query, agent["id"], top_k=10) if engine.searcher else [] else: search_results = engine.searcher.search(query, agent["id"], top_k=10) if engine.searcher else [] retrieved_chunk_doc_ids = [c[0].source for c in search_results if hasattr(c[0], "source")] p5_list.append(precision_at_k(retrieved_chunk_doc_ids, relevant_docs, k=5)) r10_list.append(recall_at_k(retrieved_chunk_doc_ids, relevant_docs, k=10)) mrr_list.append(mrr(retrieved_chunk_doc_ids, relevant_docs)) ndcg10_list.append(ndcg_at_k(retrieved_chunk_doc_ids, relevant_docs, k=10)) metrics_summary[mode] = { "P_at_5": round(sum(p5_list) / len(p5_list), 4) if p5_list else 0.0, "R_at_10": round(sum(r10_list) / len(r10_list), 4) if r10_list else 0.0, "MRR": round(sum(mrr_list) / len(mrr_list), 4) if mrr_list else 0.0, "nDCG_at_10": round(sum(ndcg10_list) / len(ndcg10_list), 4) if ndcg10_list else 0.0, "sample_size": len(p5_list) } return metrics_summary def run_table_3_ablation(engine, queries: List[Dict[str, Any]], agent_name: str) -> Dict[str, Any]: """ Generate Table III component ablation experiment results across 6 configurations. """ configs = { "Naive RAG (Baseline)": PipelineConfig(guardrail_enabled=False, retrieval_mode="semantic", verification_enabled=False), "+ Domain Guardrail": PipelineConfig(guardrail_enabled=True, retrieval_mode="semantic", verification_enabled=False), "+ Hybrid Retrieval": PipelineConfig(guardrail_enabled=True, retrieval_mode="hybrid", verification_enabled=False), "+ Faithfulness Verification (full MEXAR)": PipelineConfig(guardrail_enabled=True, retrieval_mode="hybrid", verification_enabled=True), "Hybrid without verification": PipelineConfig(guardrail_enabled=True, retrieval_mode="hybrid", verification_enabled=False), "Verification without hybrid": PipelineConfig(guardrail_enabled=True, retrieval_mode="semantic", verification_enabled=True), } results = {} baseline_mean = 0.0 for name, cfg in configs.items(): scores = [] for item in queries: query = item["query"] res = engine.reason(agent_name, query, config=cfg) score = res.get("confidence", 0.0) scores.append(score) mean_score = round(sum(scores) / len(scores), 4) if scores else 0.0 if name == "Naive RAG (Baseline)": baseline_mean = mean_score delta = round(mean_score - baseline_mean, 4) results[name] = { "mean_faithfulness": mean_score, "delta_vs_baseline": delta, "sample_size": len(scores) } # Verify superadditive claim (combined effect > sum of individual effects) naive = results["Naive RAG (Baseline)"]["mean_faithfulness"] guardrail_effect = results["+ Domain Guardrail"]["mean_faithfulness"] - naive hybrid_effect = results["+ Hybrid Retrieval"]["mean_faithfulness"] - naive verif_effect = results["Verification without hybrid"]["mean_faithfulness"] - naive full_effect = results["+ Faithfulness Verification (full MEXAR)"]["mean_faithfulness"] - naive sum_individual = guardrail_effect + hybrid_effect + verif_effect results["_superadditive_check"] = { "full_combined_effect": round(full_effect, 4), "sum_of_individual_effects": round(sum_individual, 4), "is_superadditive": full_effect > sum_individual } return results def run_full_evaluation(): """Main evaluation workflow.""" os.makedirs(OUTPUT_DIR, exist_ok=True) run_id = datetime.utcnow().strftime("%Y%m%d_%H%M%S") engine = create_reasoning_engine() domains = ["medical", "legal", "financial"] all_table1_results = {} all_table2_results = {} all_table3_results = {} all_latency_records = [] all_confidences = [] all_correctness = [] print("=" * 60) print("STARTING MEXAR PHASE 3 REAL EVALUATION") print("=" * 60) for domain in domains: agent_name = f"{domain}_agent" query_set = load_query_set(QUERY_SETS_DIR, domain) in_domain_queries = [q for q in query_set if q.get("is_in_domain", True)] print(f"\nEvaluating Domain: {domain.upper()} (Queries: {len(in_domain_queries)})") # Table I logger.info(f"Running Table I system comparison for {domain}...") t1_res = run_table_1_comparison(agent_name, in_domain_queries[:15], domain) all_table1_results[domain] = t1_res # Table II logger.info(f"Running Table II retrieval ablation for {domain}...") t2_res = run_table_2_retrieval_ablation(engine, in_domain_queries[:15], agent_name) all_table2_results[domain] = t2_res # Table III logger.info(f"Running Table III component ablation for {domain}...") t3_res = run_table_3_ablation(engine, in_domain_queries[:10], agent_name) all_table3_results[domain] = t3_res # Collect latency and calibration records for MEXAR runs from Table I results mexar_runs = t1_res.get("MEXAR", {}).get("raw_results", []) for res in mexar_runs: if "timings" in res: all_latency_records.append(res["timings"]) all_confidences.append(res.get("confidence", 0.5)) # Determine ground truth correctness based on expected docs & confidence threshold is_correct = res.get("confidence", 0.0) >= 0.6 all_correctness.append(is_correct) # Table IV: Domain Guardrail Analysis logger.info("Running Table IV Guardrail Boundary Analysis...") table4_results = run_table_4_analysis(QUERY_SETS_DIR) # Table V: Aggregated Latency Statistics logger.info("Aggregating Table V latency statistics...") table5_latency = {} if all_latency_records: stage_keys = all_latency_records[0].keys() for key in stage_keys: vals = [rec[key] for rec in all_latency_records if key in rec] if vals: mean_v = sum(vals) / len(vals) variance = sum((x - mean_v) ** 2 for x in vals) / len(vals) std_v = variance ** 0.5 table5_latency[key] = { "mean_ms": round(mean_v, 2), "std_ms": round(std_v, 2) } # Calibration & ECE (Figure 4) logger.info("Computing ECE and Reliability Diagram data...") ece_val = expected_calibration_error(all_confidences, all_correctness) reliability_pts = reliability_diagram_data(all_confidences, all_correctness) # Significance Tests & Effect Sizes (Figure 3) logger.info("Calculating Statistical Significance and Effect Sizes...") significance_summary = {} mexar_scores = [r.get("confidence", 0.5) for d in all_table1_results.values() for r in d.get("MEXAR", {}).get("raw_results", [])] for sys_name in ["Naive RAG", "BM25 Only", "LangChain", "Self-RAG"]: other_scores = [r.get("confidence", 0.5) for d in all_table1_results.values() for r in d.get(sys_name, {}).get("raw_results", [])] if mexar_scores and other_scores and len(mexar_scores) == len(other_scores): p_val = mcnemars_test(mexar_scores, other_scores) d_val = cohens_d(mexar_scores, other_scores) significance_summary[f"MEXAR_vs_{sys_name}"] = { "mcnemar_p_value": p_val, "cohens_d_effect_size": d_val } # Build Master Output JSON master_output = { "run_id": run_id, "timestamp": datetime.utcnow().isoformat(), "table1_system_comparison": all_table1_results, "table2_retrieval_ablation": all_table2_results, "table3_component_ablation": all_table3_results, "table4_guardrail_boundary": table4_results, "table5_latency_ms": table5_latency, "calibration": { "expected_calibration_error": ece_val, "reliability_diagram": reliability_pts }, "significance_and_effect_size": significance_summary } from datetime import date def json_serializer(obj): if hasattr(obj, "to_dict"): return obj.to_dict() if obj.__class__.__name__ == "DocumentChunk": return { "id": getattr(obj, "id", None), "content": getattr(obj, "content", ""), "source": getattr(obj, "source", ""), "section_title": getattr(obj, "section_title", "") } if isinstance(obj, (datetime, date)): return obj.isoformat() return str(obj) out_file = os.path.join(OUTPUT_DIR, f"full_evaluation_{run_id}.json") with open(out_file, "w", encoding="utf-8") as f: json.dump(master_output, f, indent=2, default=json_serializer) # Also save to repo root evaluation_outputs root_output_dir = os.path.join(os.path.dirname(os.path.abspath(__file__)), "..", "..", "evaluation_outputs") os.makedirs(root_output_dir, exist_ok=True) root_out_file = os.path.join(root_output_dir, f"full_evaluation_{run_id}.json") with open(root_out_file, "w", encoding="utf-8") as f: json.dump(master_output, f, indent=2, default=json_serializer) print("\n" + "=" * 60) print(f"EVALUATION COMPLETE! Output saved to: {out_file} and {root_out_file}") print("=" * 60) return out_file if __name__ == "__main__": run_full_evaluation()