feat: harden evaluation workflows and docs

README.md

@@ -140,6 +140,40 @@ npm start
 
 ---
 
+## 📈 Evaluation Workflows
+
+The scripts in `backend/evaluation` support baseline comparison, guardrail checks, benchmark runs, and ablation studies.
+
+Run from project root:
+
+```bash
+cd backend
+
+# Baseline comparison: MEXAR vs CRAG vs RAPTOR
+python evaluation/baseline_runner.py
+
+# Backbone comparison (restores original backbone after completion)
+python evaluation/backbone_comparison.py
+
+# Guardrail boundary query analysis
+python evaluation/guardrail_analysis.py
+
+# Benchmark dataset run (all rows by default) + save report
+python evaluation/benchmark_runner.py --dataset-path ../test_data/medqa_sample.json --agent-name medical_agent --output evaluation_outputs/medqa_report.json
+
+# Quick benchmark smoke test
+python evaluation/benchmark_runner.py --dataset-path ../test_data/medqa_sample.json --agent-name medical_agent --max-samples 25
+
+# McNemar significance helper
+python evaluation/statistical_tests.py
+```
+
+Notes:
+- Faithfulness values are read from `explainability.confidence_breakdown.faithfulness` when available.
+- Benchmark reports include per-query status and aggregate summary metrics.
+
+---
+
 ## 📁 Project Structure
 
 ```

backend/evaluation/ablation_chunk_size.py

@@ -7,9 +7,11 @@ sys.path.append(os.path.dirname(os.path.dirname(os.path.abspath(__file__))))
 
 from modules.knowledge_compiler import create_knowledge_compiler
 from modules.reasoning_engine import create_reasoning_engine
+from evaluation.metrics import MetricsRunner
 
 def run_chunk_ablation(agent_name: str, parsed_data: list, system_prompt: str, prompt_analysis: dict, test_queries: list):
     sizes = [64, 128, 256, 512, 1024]
+    metrics = MetricsRunner()
     
     for size in sizes:
         print(f"\n=====================")
@@ -28,8 +30,12 @@ def run_chunk_ablation(agent_name: str, parsed_data: list, system_prompt: str, p
             engine = create_reasoning_engine()
             for q in test_queries:
                 res = engine.reason(agent_name, q)
+                faithfulness = metrics.extract_faithfulness(res)
                 print(f"Q: {q}")
-                print(f"Faithfulness: {res['explainability']['faithfulness']}")
+                if faithfulness is None:
+                    print("Faithfulness: N/A")
+                else:
+                    print(f"Faithfulness: {faithfulness:.3f}")
         except Exception as e:
             print(f"Failed ablation step for size {size}: {e}")
 

backend/evaluation/backbone_comparison.py

@@ -7,24 +7,35 @@ sys.path.append(os.path.dirname(os.path.dirname(os.path.abspath(__file__))))
 
 from core.config import settings
 from modules.reasoning_engine import create_reasoning_engine
+from evaluation.metrics import MetricsRunner
 
 def run_comparison(agent_name: str, queries: list):
     backbones = ["llama3", "mixtral", "gemma"]
-    
-    for bb in backbones:
-        settings.LLM_BACKBONE = bb
-        print(f"\n--- Testing Backbone: {bb} ---")
-        try:
-            # Must recreate engine so GroqClient picks up config
-            engine = create_reasoning_engine()
-            
-            for q in queries:
-                res = engine.reason(agent_name, q)
-                print(f"Q: {q}")
-                print(f"A ({bb}): {res['answer'][:100]}...")
-                print(f"Faithfulness: {res['explainability']['faithfulness']}")
-        except Exception as e:
-            print(f"Failed to run with backbone {bb}: {e}")
+    metrics = MetricsRunner()
+    original_backbone = getattr(settings, "LLM_BACKBONE", None)
+
+    try:
+        for bb in backbones:
+            settings.LLM_BACKBONE = bb
+            print(f"\n--- Testing Backbone: {bb} ---")
+            try:
+                # Must recreate engine so GroqClient picks up config
+                engine = create_reasoning_engine()
+
+                for q in queries:
+                    res = engine.reason(agent_name, q)
+                    faithfulness = metrics.extract_faithfulness(res)
+                    print(f"Q: {q}")
+                    print(f"A ({bb}): {res['answer'][:100]}...")
+                    if faithfulness is None:
+                        print("Faithfulness: N/A")
+                    else:
+                        print(f"Faithfulness: {faithfulness:.3f}")
+            except Exception as e:
+                print(f"Failed to run with backbone {bb}: {e}")
+    finally:
+        settings.LLM_BACKBONE = original_backbone
+        print(f"\nRestored LLM_BACKBONE to: {original_backbone}")
 
 if __name__ == "__main__":
     test_queries = ["What are the symptoms of a common cold?"]

backend/evaluation/baseline_runner.py

@@ -3,43 +3,70 @@ Runs CRAG and RAPTOR baselines against a set of test queries.
 """
 import sys
 import os
+from typing import Dict, List, Optional
+
 sys.path.append(os.path.dirname(os.path.dirname(os.path.abspath(__file__))))
 
 from modules.reasoning_engine import create_reasoning_engine
 from evaluation.metrics import MetricsRunner
 
-def run_baselines(agent_name: str, queries: list):
+
+def _append_score(results: Dict[str, List[float]], baseline: str, score: Optional[float]) -> None:
+    if score is None:
+        print(f"{baseline}: Faithfulness score unavailable for this query.")
+        return
+    results[baseline].append(score)
+
+
+def run_baselines(agent_name: str, queries: List[str]):
     engine = create_reasoning_engine()
     metrics = MetricsRunner()
-    
-    results = {"CRAG": [], "RAPTOR": [], "MEXAR": []}
-    
+
+    results: Dict[str, List[float]] = {"CRAG": [], "RAPTOR": [], "MEXAR": []}
+
     for q in queries:
         print(f"\nProcessing query: {q}")
-        
+
         try:
             # Original MEXAR
             res_mexar = engine.reason(agent_name, q)
-            results["MEXAR"].append(float(res_mexar["explainability"]["faithfulness"].strip('%'))/100)
-            
+            mexar_score = metrics.extract_faithfulness(res_mexar)
+            _append_score(results, "MEXAR", mexar_score)
+
             # CRAG
             res_crag = engine.reason_crag_baseline(agent_name, q)
-            results["CRAG"].append(res_crag["confidence"]) # The raw score
-            
+            crag_score = metrics.extract_faithfulness(res_crag)
+            if crag_score is None:
+                crag_score = metrics.extract_confidence(res_crag)
+            _append_score(results, "CRAG", crag_score)
+
             # RAPTOR
             res_raptor = engine.reason_raptor_baseline(agent_name, q)
-            results["RAPTOR"].append(res_raptor["confidence"])
+            raptor_score = metrics.extract_faithfulness(res_raptor)
+            if raptor_score is None:
+                raptor_score = metrics.extract_confidence(res_raptor)
+            _append_score(results, "RAPTOR", raptor_score)
+
+            print(
+                "Scores -> "
+                f"MEXAR: {mexar_score if mexar_score is not None else 'N/A'}, "
+                f"CRAG: {crag_score if crag_score is not None else 'N/A'}, "
+                f"RAPTOR: {raptor_score if raptor_score is not None else 'N/A'}"
+            )
         except Exception as e:
             print(f"Error evaluating query '{q}': {e}")
-        
+
     print("\n--- Baseline Comparison (Faithfulness) ---")
-    for b_name in results:
-        if results[b_name]:
-            avg = sum(results[b_name]) / len(results[b_name])
-            print(f"{b_name}: {avg:.4f}")
+    for b_name, scores in results.items():
+        if scores:
+            avg = sum(scores) / len(scores)
+            print(f"{b_name}: {avg:.4f} (n={len(scores)})")
         else:
             print(f"{b_name}: No results")
 
+    return results
+
+
 if __name__ == "__main__":
     # Example usage
     test_queries = [

backend/evaluation/benchmark_runner.py

@@ -4,32 +4,167 @@ Runs evaluation on public benchmarks like MedQA, LegalBench.
 import sys
 import os
 import json
+import argparse
+from datetime import datetime
+from typing import Any, Dict, List, Optional
+
 sys.path.append(os.path.dirname(os.path.dirname(os.path.abspath(__file__))))
 
 from modules.reasoning_engine import create_reasoning_engine
+from evaluation.metrics import MetricsRunner
+
+
+def _extract_query(item: Dict[str, Any]) -> Optional[str]:
+    query = item.get("question") or item.get("query")
+    if not isinstance(query, str):
+        return None
+    query = query.strip()
+    return query if query else None
+
 
-def run_benchmark(dataset_path: str, agent_name: str):
+def _summarize_scores(scores: List[float]) -> Optional[float]:
+    if not scores:
+        return None
+    return round(sum(scores) / len(scores), 4)
+
+
+def run_benchmark(
+    dataset_path: str,
+    agent_name: str,
+    max_samples: Optional[int] = None,
+    output_path: Optional[str] = None,
+) -> Dict[str, Any]:
     engine = create_reasoning_engine()
-    
+    metrics = MetricsRunner()
+
     if not os.path.exists(dataset_path):
-        print(f"Dataset not found: {dataset_path}")
-        return
-        
-    with open(dataset_path, "r") as f:
+        raise FileNotFoundError(f"Dataset not found: {dataset_path}")
+
+    with open(dataset_path, "r", encoding="utf-8") as f:
         data = json.load(f)
-        
-    for item in data[:10]: # Run first 10 for demo
-        query = item.get("question") or item.get("query")
+
+    if not isinstance(data, list):
+        raise ValueError("Benchmark dataset must be a JSON array of records")
+
+    items = data if not max_samples else data[:max_samples]
+
+    records: List[Dict[str, Any]] = []
+    faithfulness_scores: List[float] = []
+    succeeded = 0
+    failed = 0
+    skipped = 0
+
+    for idx, item in enumerate(items, start=1):
+        query = _extract_query(item)
         if not query:
+            skipped += 1
             continue
-            
-        print(f"\nQuery: {query}")
+
+        print(f"\n[{idx}/{len(items)}] Query: {query}")
+        row: Dict[str, Any] = {
+            "index": idx,
+            "query": query,
+        }
+
         try:
             result = engine.reason(agent_name, query)
-            print(f"Answer: {result['answer'][:100]}...")
-            print(f"Faithfulness: {result['explainability']['faithfulness']}")
+            faithfulness = metrics.extract_faithfulness(result)
+            confidence = metrics.extract_confidence(result)
+            answer = result.get("answer", "")
+
+            if isinstance(answer, str) and len(answer) > 120:
+                answer_preview = f"{answer[:120]}..."
+            else:
+                answer_preview = answer
+
+            row.update({
+                "status": "ok",
+                "in_domain": result.get("in_domain"),
+                "confidence": confidence,
+                "faithfulness": faithfulness,
+                "answer_preview": answer_preview,
+            })
+            records.append(row)
+
+            if faithfulness is not None:
+                faithfulness_scores.append(faithfulness)
+            succeeded += 1
+
+            print(f"Answer: {answer_preview}")
+            if faithfulness is None:
+                print("Faithfulness: N/A")
+            else:
+                print(f"Faithfulness: {faithfulness:.3f}")
         except Exception as e:
+            row.update({
+                "status": "error",
+                "error": str(e),
+            })
+            records.append(row)
+            failed += 1
             print(f"Failed to process query: {e}")
 
+    summary: Dict[str, Any] = {
+        "dataset_path": dataset_path,
+        "agent_name": agent_name,
+        "total_rows": len(data),
+        "attempted_rows": len(items),
+        "succeeded": succeeded,
+        "failed": failed,
+        "skipped": skipped,
+        "avg_faithfulness": _summarize_scores(faithfulness_scores),
+        "generated_at_utc": datetime.utcnow().isoformat() + "Z",
+    }
+
+    print("\n--- Benchmark Summary ---")
+    print(f"Attempted: {summary['attempted_rows']}")
+    print(f"Succeeded: {summary['succeeded']}")
+    print(f"Failed: {summary['failed']}")
+    print(f"Skipped: {summary['skipped']}")
+    print(f"Avg faithfulness: {summary['avg_faithfulness']}")
+
+    if output_path:
+        output_dir = os.path.dirname(output_path)
+        if output_dir:
+            os.makedirs(output_dir, exist_ok=True)
+        payload = {
+            "summary": summary,
+            "results": records,
+        }
+        with open(output_path, "w", encoding="utf-8") as f:
+            json.dump(payload, f, indent=2)
+        print(f"Saved report to: {output_path}")
+
+    return {
+        "summary": summary,
+        "results": records,
+    }
+
+
+def _default_dataset_path() -> str:
+    return os.path.join(
+        os.path.dirname(os.path.dirname(os.path.dirname(__file__))),
+        "test_data",
+        "medqa_sample.json",
+    )
+
+
+def parse_args() -> argparse.Namespace:
+    parser = argparse.ArgumentParser(description="Run benchmark dataset evaluation")
+    parser.add_argument("--dataset-path", default=_default_dataset_path(), help="Path to benchmark JSON file")
+    parser.add_argument("--agent-name", default="medical_agent", help="Compiled agent name")
+    parser.add_argument(
+        "--max-samples",
+        type=int,
+        default=0,
+        help="Limit to first N records (0 means all)",
+    )
+    parser.add_argument("--output", default="", help="Optional output path for JSON report")
+    return parser.parse_args()
+
+
 if __name__ == "__main__":
-    run_benchmark(os.path.join(os.path.dirname(os.path.dirname(os.path.dirname(__file__))), "test_data", "medqa_sample.json"), "medical_agent")
+    args = parse_args()
+    max_samples = args.max_samples if args.max_samples > 0 else None
+    output_path = args.output if args.output else None
+    run_benchmark(args.dataset_path, args.agent_name, max_samples=max_samples, output_path=output_path)

backend/evaluation/metrics.py

@@ -4,17 +4,20 @@ Calculates common metrics across different baselines and experiments.
 """
 import sys
 import os
+from typing import Any, Dict, Optional
+
 sys.path.append(os.path.dirname(os.path.dirname(os.path.abspath(__file__))))
 
 from utils.faithfulness import FaithfulnessScorer, BartNLIScorer, FActScoreCompat
 
+
 class MetricsRunner:
     def __init__(self):
         self.faith_scorer = FaithfulnessScorer()
         self.bart_nli = BartNLIScorer()
         self.factscore = FActScoreCompat()
 
-    def evaluate_all(self, answer: str, context: str):
+    def evaluate_all(self, answer: str, context: str) -> Dict[str, float]:
         faith_res = self.faith_scorer.score(answer, context)
         bart_res = self.bart_nli.score(answer, context)
         fact_res = self.factscore.score(answer, context)
@@ -23,3 +26,62 @@ class MetricsRunner:
             "bart_nli": bart_res.score,
             "factscore": fact_res.score
         }
+
+    def extract_faithfulness(self, response: Dict[str, Any]) -> Optional[float]:
+        """Extract faithfulness score from response payloads across formats."""
+        if not isinstance(response, dict):
+            return None
+
+        explainability = response.get("explainability") or {}
+        confidence_breakdown = explainability.get("confidence_breakdown") or {}
+
+        for candidate in (
+            confidence_breakdown.get("faithfulness"),
+            explainability.get("faithfulness"),
+        ):
+            parsed = self._parse_numeric(candidate)
+            if parsed is not None:
+                return self._clamp(parsed)
+
+        return None
+
+    def extract_confidence(self, response: Dict[str, Any]) -> Optional[float]:
+        """Extract numeric confidence score if available."""
+        if not isinstance(response, dict):
+            return None
+
+        parsed = self._parse_numeric(response.get("confidence"))
+        if parsed is None:
+            return None
+        return self._clamp(parsed)
+
+    @staticmethod
+    def _clamp(value: float) -> float:
+        return max(0.0, min(1.0, value))
+
+    @staticmethod
+    def _parse_numeric(value: Any) -> Optional[float]:
+        if value is None:
+            return None
+
+        if isinstance(value, (int, float)):
+            return float(value)
+
+        if isinstance(value, str):
+            cleaned = value.strip()
+            if not cleaned:
+                return None
+
+            if cleaned.endswith("%"):
+                cleaned = cleaned[:-1].strip()
+                try:
+                    return float(cleaned) / 100.0
+                except ValueError:
+                    return None
+
+            try:
+                return float(cleaned)
+            except ValueError:
+                return None
+
+        return None