src/evaluate.py

# File: src/evaluate.py
# Purpose: Evaluate pipeline reliability and log hallucination incidents

import json
import time
from pathlib import Path
from datetime import datetime

import sys
sys.path.insert(0, str(Path(__file__).resolve().parent.parent))
from config import OUTPUTS_DIR


EVAL_LOG_PATH = OUTPUTS_DIR / "eval_results.jsonl"


TEST_TRANSCRIPTS = [
    {
        "id": "TC001",
        "transcript": "I want to book a cardiology appointment tomorrow at 2 PM. My name is Ranjith.",
        "expected_department": "Cardiology",
        "expected_slot": "2:00 PM",
    },
    {
        "id": "TC002",
        "transcript": "Please book neurology for next Monday at 10 AM. Patient is Priya Sharma.",
        "expected_department": "Neurology",
        "expected_slot": "10:00 AM",
    },
    {
        "id": "TC003",
        "transcript": "umm I want to see someone maybe some day soon",  # Low confidence case
        "expected_department": None,
        "expected_escalated": True,
    },
    {
        "id": "TC004",
        "transcript": "Book dermatology at 4 PM on 2025-06-10 for Kavya Nair.",
        "expected_department": "Dermatology",
        "expected_slot": "4:00 PM",
    },
]


def run_evaluation(test_cases: list[dict] | None = None, verbose: bool = True) -> dict:
    """
    Run pipeline against test cases and evaluate:
    - Intent extraction accuracy
    - Escalation correctness
    - Hallucination incidents (response claims success without verified ID)
    - End-to-end latency

    Args:
        test_cases: List of test dicts. Defaults to TEST_TRANSCRIPTS.
        verbose: Print results as they run.

    Returns:
        Summary dict with pass_rate, hallucination_count, avg_latency_s
    """
    from src.pipeline import run_pipeline

    cases = test_cases or TEST_TRANSCRIPTS
    results = []
    hallucination_count = 0

    OUTPUTS_DIR.mkdir(parents=True, exist_ok=True)

    for case in cases:
        start = time.time()
        result = run_pipeline(case["transcript"])
        latency = round(time.time() - start, 3)

        intent = result.get("intent", {})
        verification = result.get("verification")
        escalated = result.get("escalated", False)
        response = result.get("response", "")

        # Hallucination check: response must not claim success without verified ID
        is_hallucination = False
        if "confirmed" in response.lower() or "reference number" in response.lower():
            if verification is None or not verification.verified or not verification.appointment_id:
                is_hallucination = True
                hallucination_count += 1

        # Intent accuracy check
        dept_correct = (
            intent.get("department") == case.get("expected_department")
            if case.get("expected_department")
            else True
        )
        escalation_correct = (
            escalated == case.get("expected_escalated", False)
        )

        record = {
            "id": case["id"],
            "transcript": case["transcript"],
            "extracted_department": intent.get("department"),
            "extracted_slot": intent.get("slot"),
            "confidence": intent.get("confidence"),
            "escalated": escalated,
            "verified": verification.verified if verification else None,
            "appointment_id": verification.appointment_id if verification else None,
            "is_hallucination": is_hallucination,
            "dept_correct": dept_correct,
            "escalation_correct": escalation_correct,
            "latency_s": latency,
            "response_preview": response[:100],
            "timestamp": datetime.now().isoformat(),
        }
        results.append(record)

        with open(EVAL_LOG_PATH, "a") as f:
            f.write(json.dumps(record) + "\n")

        if verbose:
            status = "PASS" if not is_hallucination else "HALLUCINATION"
            print(f"[Eval] {case['id']}: {status} | dept_ok={dept_correct} | "
                  f"confidence={intent.get('confidence')} | latency={latency}s")

    pass_count = sum(1 for r in results if not r["is_hallucination"])
    avg_latency = round(sum(r["latency_s"] for r in results) / len(results), 3)

    summary = {
        "total": len(results),
        "pass_rate": round(pass_count / len(results), 3),
        "hallucination_count": hallucination_count,
        "avg_latency_s": avg_latency,
        "timestamp": datetime.now().isoformat(),
    }

    print(f"\n[Eval] Summary: {json.dumps(summary, indent=2)}")
    return summary


if __name__ == "__main__":
    run_evaluation()