eval/eval_harness.py

"""
Evaluation harness: measures Recall@10 and behavior probes against
the provided sample conversation traces.

This is what SHL reviewers care about most — measured, reproducible metrics.
"""

import json
import re
import httpx
import time
from pathlib import Path


BASE_URL = "http://localhost:8000"
TRACES_DIR = Path(__file__).parent.parent / "sample_conversations" / "GenAI_SampleConversations"


def parse_trace(filepath: Path) -> dict:
    """
    Parse a conversation trace markdown file.
    Extracts: user messages, expected recommendations (from last turn with table).
    """
    text = filepath.read_text(encoding="utf-8")
    
    # Extract all turns
    turns = []
    turn_blocks = re.split(r'###\s+Turn\s+\d+', text)
    
    for block in turn_blocks[1:]:  # Skip the header
        user_match = re.search(r'\*\*User\*\*\s*\n\s*>\s*(.+?)(?:\n\n|\n\*\*)', block, re.DOTALL)
        user_msg = ""
        if user_match:
            user_msg = user_match.group(1).strip()
            # Clean up multi-line quotes
            user_msg = re.sub(r'\n\s*>\s*', '\n', user_msg).strip()
        
        turns.append({
            "user_message": user_msg,
            "block": block,
        })
    
    # Extract expected recommendations from the LAST table in the trace
    expected_recs = []
    table_pattern = re.compile(
        r'\|\s*\d+\s*\|\s*(.+?)\s*\|\s*\S+\s*\|\s*.+?\s*\|\s*.+?\s*\|\s*.+?\s*\|\s*<?(https://www\.shl\.com/.+?)>?\s*\|',
        re.MULTILINE,
    )
    
    for match in table_pattern.finditer(text):
        name = match.group(1).strip()
        url = match.group(2).strip()
        expected_recs.append({"name": name, "url": url})
    
    # Deduplicate — keep the latest set (from the final turn)
    # Find the last turn that has a table
    last_table_recs = []
    for turn in reversed(turns):
        table_matches = table_pattern.finditer(turn["block"])
        recs_in_turn = []
        for m in table_matches:
            recs_in_turn.append({"name": m.group(1).strip(), "url": m.group(2).strip()})
        if recs_in_turn:
            last_table_recs = recs_in_turn
            break
    
    if last_table_recs:
        expected_recs = last_table_recs
    
    return {
        "filename": filepath.name,
        "turns": turns,
        "expected_recommendations": expected_recs,
    }


def run_conversation(trace: dict, base_url: str = BASE_URL) -> dict:
    """
    Run a conversation against the agent using the trace's user messages.
    Returns the agent's final recommendations and turn count.
    """
    messages = []
    last_response = None
    turn_count = 0
    
    client = httpx.Client(timeout=35.0)
    
    for turn in trace["turns"]:
        user_msg = turn["user_message"]
        if not user_msg:
            continue
        
        messages.append({"role": "user", "content": user_msg})
        
        try:
            response = client.post(
                f"{base_url}/chat",
                json={"messages": messages},
            )
            response.raise_for_status()
            last_response = response.json()
            
            # Add assistant reply to history
            messages.append({
                "role": "assistant",
                "content": last_response.get("reply", ""),
            })
            
            turn_count += 1
            
            # Check if conversation is done
            if last_response.get("end_of_conversation", False):
                break
                
        except Exception as e:
            print(f"  Error on turn {turn_count + 1}: {e}")
            break
    
    client.close()
    
    return {
        "turn_count": turn_count,
        "final_response": last_response,
        "final_recommendations": (
            last_response.get("recommendations", []) if last_response else []
        ),
    }


def compute_recall_at_k(
    predicted: list[dict],
    expected: list[dict],
    k: int = 10,
) -> float:
    """
    Compute Recall@K.
    Recall@K = (number of relevant items in top K) / (total relevant items)
    """
    if not expected:
        return 1.0  # No expected items means we can't fail
    
    expected_urls = {rec["url"] for rec in expected}
    predicted_urls = {rec.get("url", "") for rec in predicted[:k]}
    
    hits = len(expected_urls & predicted_urls)
    recall = hits / len(expected_urls)
    
    return recall


def check_schema_compliance(response: dict) -> bool:
    """Check if a response matches the required schema."""
    if not isinstance(response, dict):
        return False
    
    required_fields = ["reply", "recommendations", "end_of_conversation"]
    for field in required_fields:
        if field not in response:
            return False
    
    if not isinstance(response["reply"], str):
        return False
    
    if not isinstance(response["recommendations"], list):
        return False
    
    for rec in response["recommendations"]:
        if not isinstance(rec, dict):
            return False
        if "name" not in rec or "url" not in rec or "test_type" not in rec:
            return False
    
    if not isinstance(response["end_of_conversation"], bool):
        return False
    
    return True


def run_full_evaluation(base_url: str = BASE_URL):
    """Run the full evaluation suite and print results."""
    print("=" * 60)
    print("SHL Assessment Recommender — Evaluation Report")
    print("=" * 60)
    
    # Check health
    try:
        client = httpx.Client(timeout=120.0)
        health = client.get(f"{base_url}/health")
        print(f"\n✓ Health check: {health.status_code} — {health.json()}")
        client.close()
    except Exception as e:
        print(f"\n✗ Health check failed: {e}")
        return
    
    # Find all trace files
    trace_files = sorted(TRACES_DIR.glob("C*.md"))
    if not trace_files:
        print(f"\n✗ No trace files found in {TRACES_DIR}")
        return
    
    print(f"\nFound {len(trace_files)} conversation traces\n")
    
    results = []
    total_recall = 0.0
    total_turns = 0
    schema_passes = 0
    
    for trace_file in trace_files:
        trace = parse_trace(trace_file)
        print(f"--- {trace['filename']} ---")
        print(f"  Expected: {len(trace['expected_recommendations'])} assessments")
        
        result = run_conversation(trace, base_url)
        
        # Schema compliance
        schema_ok = check_schema_compliance(result["final_response"]) if result["final_response"] else False
        if schema_ok:
            schema_passes += 1
        
        # Recall@10
        recall = compute_recall_at_k(
            result["final_recommendations"],
            trace["expected_recommendations"],
            k=10,
        )
        total_recall += recall
        total_turns += result["turn_count"]
        
        print(f"  Turns: {result['turn_count']}")
        print(f"  Recommendations: {len(result['final_recommendations'])}")
        print(f"  Recall@10: {recall:.2f}")
        print(f"  Schema OK: {'✓' if schema_ok else '✗'}")
        
        # Print predicted vs expected
        if result["final_recommendations"]:
            pred_names = [r.get("name", "?") for r in result["final_recommendations"]]
            print(f"  Predicted: {pred_names}")
        exp_names = [r["name"] for r in trace["expected_recommendations"]]
        print(f"  Expected:  {exp_names}")
        print()
        
        results.append({
            "trace": trace["filename"],
            "recall": recall,
            "turns": result["turn_count"],
            "schema_ok": schema_ok,
        })
    
    # Summary
    n = len(results)
    print("=" * 60)
    print("SUMMARY")
    print("=" * 60)
    print(f"Mean Recall@10:      {total_recall / n:.3f}")
    print(f"Avg Turns:           {total_turns / n:.1f}")
    print(f"Schema Compliance:   {schema_passes}/{n} ({schema_passes/n*100:.0f}%)")
    print(f"Total Traces:        {n}")
    print("=" * 60)


if __name__ == "__main__":
    run_full_evaluation()