Fix fact extraction - pass raw data for simple tools.......

evaluation/__init__.py

@@ -0,0 +1,37 @@
+"""
+Evaluation Framework for Pre-Visit Summary Agent
+
+This package provides tools to evaluate the accuracy of the
+pre-visit summary agent's data retrieval and reporting.
+
+Modules:
+    test_generator: Generates test cases from Synthea database
+    expected_values: Computes ground truth values from database
+    evaluator: Compares agent facts vs expected values
+    metrics: Aggregates results and computes summary statistics
+    facts_schema: Defines structured output format for agent
+    run_evaluation: Main entry point for running evaluations
+
+Usage:
+    # Run direct evaluation (validates framework)
+    python -m evaluation.run_evaluation --mode direct --patients 10
+    
+    # Run simulated evaluation (tests error detection)
+    python -m evaluation.run_evaluation --mode simulated --error-rate 0.15
+"""
+
+from .test_generator import generate_all_test_cases, get_test_summary
+from .expected_values import compute_expected_values
+from .evaluator import evaluate_case, CaseEvaluation
+from .metrics import aggregate_metrics, format_report, EvaluationMetrics
+
+__all__ = [
+    "generate_all_test_cases",
+    "get_test_summary", 
+    "compute_expected_values",
+    "evaluate_case",
+    "CaseEvaluation",
+    "aggregate_metrics",
+    "format_report",
+    "EvaluationMetrics"
+]

evaluation/create_test_db.py

@@ -0,0 +1,291 @@
+#!/usr/bin/env python3
+"""
+Create a minimal test database for evaluation framework validation.
+
+This creates a small SQLite database with sample patient data
+that can be used to test the evaluation framework.
+"""
+
+import sqlite3
+import os
+from datetime import datetime, timedelta
+import random
+
+DB_PATH = "data/fhir.db"
+
+
+def create_test_database():
+    """Create test database with sample data."""
+    
+    os.makedirs(os.path.dirname(DB_PATH), exist_ok=True)
+    
+    conn = sqlite3.connect(DB_PATH)
+    cursor = conn.cursor()
+    
+    # Create tables
+    cursor.executescript("""
+        -- Patients table
+        CREATE TABLE IF NOT EXISTS patients (
+            id TEXT PRIMARY KEY,
+            given_name TEXT,
+            family_name TEXT,
+            birth_date TEXT,
+            gender TEXT,
+            marital_status TEXT
+        );
+        
+        -- Conditions table
+        CREATE TABLE IF NOT EXISTS conditions (
+            id TEXT PRIMARY KEY,
+            patient_id TEXT,
+            code TEXT,
+            display TEXT,
+            clinical_status TEXT,
+            onset_date TEXT,
+            abatement_date TEXT
+        );
+        
+        -- Medications table
+        CREATE TABLE IF NOT EXISTS medications (
+            id TEXT PRIMARY KEY,
+            patient_id TEXT,
+            code TEXT,
+            display TEXT,
+            status TEXT,
+            start_date TEXT
+        );
+        
+        -- Observations table
+        CREATE TABLE IF NOT EXISTS observations (
+            id TEXT PRIMARY KEY,
+            patient_id TEXT,
+            code TEXT,
+            display TEXT,
+            value_quantity REAL,
+            unit TEXT,
+            effective_date TEXT,
+            category TEXT
+        );
+        
+        -- Allergies table
+        CREATE TABLE IF NOT EXISTS allergies (
+            id TEXT PRIMARY KEY,
+            patient_id TEXT,
+            substance TEXT,
+            reaction_display TEXT,
+            criticality TEXT,
+            category TEXT
+        );
+        
+        -- Immunizations table
+        CREATE TABLE IF NOT EXISTS immunizations (
+            id TEXT PRIMARY KEY,
+            patient_id TEXT,
+            vaccine_code TEXT,
+            vaccine_display TEXT,
+            status TEXT,
+            occurrence_date TEXT
+        );
+        
+        -- Procedures table
+        CREATE TABLE IF NOT EXISTS procedures (
+            id TEXT PRIMARY KEY,
+            patient_id TEXT,
+            code TEXT,
+            display TEXT,
+            status TEXT,
+            performed_date TEXT
+        );
+        
+        -- Encounters table
+        CREATE TABLE IF NOT EXISTS encounters (
+            id TEXT PRIMARY KEY,
+            patient_id TEXT,
+            status TEXT,
+            class_code TEXT,
+            class_display TEXT,
+            type_code TEXT,
+            type_display TEXT,
+            reason_code TEXT,
+            reason_display TEXT,
+            period_start TEXT,
+            period_end TEXT
+        );
+    """)
+    
+    # Create test patients
+    patients = [
+        ("patient-001", "John", "Smith", "1965-03-15", "male"),
+        ("patient-002", "Mary", "Johnson", "1978-07-22", "female"),
+        ("patient-003", "Robert", "Williams", "1952-11-08", "male"),
+    ]
+    
+    for pid, given, family, dob, gender in patients:
+        cursor.execute("""
+            INSERT OR REPLACE INTO patients (id, given_name, family_name, birth_date, gender)
+            VALUES (?, ?, ?, ?, ?)
+        """, (pid, given, family, dob, gender))
+    
+    # Create conditions
+    conditions = [
+        ("patient-001", "44054006", "Type 2 Diabetes Mellitus", "active", "2015-06-10"),
+        ("patient-001", "38341003", "Essential Hypertension", "active", "2018-02-15"),
+        ("patient-002", "195967001", "Asthma", "active", "2010-04-20"),
+        ("patient-002", "73211009", "Type 2 Diabetes Mellitus", "active", "2020-01-10"),
+        ("patient-003", "38341003", "Essential Hypertension", "active", "2005-08-12"),
+        ("patient-003", "13644009", "Hypercholesterolemia", "active", "2010-03-25"),
+    ]
+    
+    for i, (pid, code, display, status, onset) in enumerate(conditions):
+        cursor.execute("""
+            INSERT OR REPLACE INTO conditions (id, patient_id, code, display, clinical_status, onset_date)
+            VALUES (?, ?, ?, ?, ?, ?)
+        """, (f"cond-{i+1:03d}", pid, code, display, status, onset))
+    
+    # Create medications
+    medications = [
+        ("patient-001", "860975", "Metformin 500 MG Oral Tablet", "active", "2015-06-15"),
+        ("patient-001", "314076", "Lisinopril 10 MG Oral Tablet", "active", "2018-02-20"),
+        ("patient-002", "895994", "Albuterol 90 MCG Inhaler", "active", "2010-05-01"),
+        ("patient-002", "860975", "Metformin 500 MG Oral Tablet", "active", "2020-01-15"),
+        ("patient-003", "314076", "Lisinopril 20 MG Oral Tablet", "active", "2005-08-20"),
+        ("patient-003", "316672", "Atorvastatin 20 MG Oral Tablet", "active", "2010-04-01"),
+    ]
+    
+    for i, (pid, code, display, status, start) in enumerate(medications):
+        cursor.execute("""
+            INSERT OR REPLACE INTO medications (id, patient_id, code, display, status, start_date)
+            VALUES (?, ?, ?, ?, ?, ?)
+        """, (f"med-{i+1:03d}", pid, code, display, status, start))
+    
+    # Create observations (vitals)
+    base_date = datetime.now()
+    
+    for pid in ["patient-001", "patient-002", "patient-003"]:
+        obs_id = 1
+        
+        # Blood pressure readings over last 30 days
+        for days_ago in range(0, 30, 5):
+            date = (base_date - timedelta(days=days_ago)).strftime("%Y-%m-%d")
+            systolic = random.randint(120, 145)
+            diastolic = random.randint(75, 95)
+            
+            cursor.execute("""
+                INSERT OR REPLACE INTO observations 
+                (id, patient_id, code, display, value_quantity, unit, effective_date, category)
+                VALUES (?, ?, ?, ?, ?, ?, ?, ?)
+            """, (f"obs-{pid}-{obs_id}", pid, "8480-6", "Systolic Blood Pressure", 
+                  systolic, "mmHg", date, "vital-signs"))
+            obs_id += 1
+            
+            cursor.execute("""
+                INSERT OR REPLACE INTO observations 
+                (id, patient_id, code, display, value_quantity, unit, effective_date, category)
+                VALUES (?, ?, ?, ?, ?, ?, ?, ?)
+            """, (f"obs-{pid}-{obs_id}", pid, "8462-4", "Diastolic Blood Pressure",
+                  diastolic, "mmHg", date, "vital-signs"))
+            obs_id += 1
+            
+            # Heart rate
+            hr = random.randint(65, 85)
+            cursor.execute("""
+                INSERT OR REPLACE INTO observations 
+                (id, patient_id, code, display, value_quantity, unit, effective_date, category)
+                VALUES (?, ?, ?, ?, ?, ?, ?, ?)
+            """, (f"obs-{pid}-{obs_id}", pid, "8867-4", "Heart Rate",
+                  hr, "/min", date, "vital-signs"))
+            obs_id += 1
+        
+        # A1c readings (quarterly)
+        for months_ago in [0, 3, 6, 9]:
+            date = (base_date - timedelta(days=months_ago*30)).strftime("%Y-%m-%d")
+            a1c = round(random.uniform(6.0, 8.5), 1)
+            
+            cursor.execute("""
+                INSERT OR REPLACE INTO observations 
+                (id, patient_id, code, display, value_quantity, unit, effective_date, category)
+                VALUES (?, ?, ?, ?, ?, ?, ?, ?)
+            """, (f"obs-{pid}-{obs_id}", pid, "4548-4", "Hemoglobin A1c",
+                  a1c, "%", date, "laboratory"))
+            obs_id += 1
+    
+    # Create allergies
+    allergies = [
+        ("patient-001", "Penicillin", "Hives", "high", "medication"),
+        ("patient-002", "Peanuts", "Anaphylaxis", "high", "food"),
+        ("patient-002", "Latex", "Rash", "low", "environment"),
+        ("patient-003", "Sulfa drugs", "Rash", "moderate", "medication"),
+    ]
+    
+    for i, (pid, substance, reaction, criticality, category) in enumerate(allergies):
+        cursor.execute("""
+            INSERT OR REPLACE INTO allergies 
+            (id, patient_id, substance, reaction_display, criticality, category)
+            VALUES (?, ?, ?, ?, ?, ?)
+        """, (f"allergy-{i+1:03d}", pid, substance, reaction, criticality, category))
+    
+    # Create immunizations
+    immunizations = [
+        ("patient-001", "140", "Influenza Vaccine", "completed", "2024-10-15"),
+        ("patient-001", "207", "COVID-19 Vaccine", "completed", "2024-01-20"),
+        ("patient-002", "140", "Influenza Vaccine", "completed", "2024-11-01"),
+        ("patient-002", "113", "Tdap Vaccine", "completed", "2022-05-10"),
+        ("patient-003", "140", "Influenza Vaccine", "completed", "2024-09-20"),
+        ("patient-003", "33", "Pneumococcal Vaccine", "completed", "2023-03-15"),
+    ]
+    
+    for i, (pid, code, display, status, date) in enumerate(immunizations):
+        cursor.execute("""
+            INSERT OR REPLACE INTO immunizations 
+            (id, patient_id, vaccine_code, vaccine_display, status, occurrence_date)
+            VALUES (?, ?, ?, ?, ?, ?)
+        """, (f"imm-{i+1:03d}", pid, code, display, status, date))
+    
+    # Create procedures
+    procedures = [
+        ("patient-001", "73761001", "Colonoscopy", "completed", "2023-06-15"),
+        ("patient-002", "80146002", "Appendectomy", "completed", "2015-08-20"),
+        ("patient-003", "232717009", "Coronary Angioplasty", "completed", "2020-02-10"),
+        ("patient-003", "73761001", "Colonoscopy", "completed", "2022-04-05"),
+    ]
+    
+    for i, (pid, code, display, status, date) in enumerate(procedures):
+        cursor.execute("""
+            INSERT OR REPLACE INTO procedures 
+            (id, patient_id, code, display, status, performed_date)
+            VALUES (?, ?, ?, ?, ?, ?)
+        """, (f"proc-{i+1:03d}", pid, code, display, status, date))
+    
+    # Create encounters
+    for pid in ["patient-001", "patient-002", "patient-003"]:
+        for i in range(5):
+            days_ago = i * 60  # Every ~2 months
+            start = (base_date - timedelta(days=days_ago)).strftime("%Y-%m-%d")
+            end = start
+            
+            cursor.execute("""
+                INSERT OR REPLACE INTO encounters 
+                (id, patient_id, status, class_code, class_display, type_code, type_display,
+                 reason_code, reason_display, period_start, period_end)
+                VALUES (?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?)
+            """, (f"enc-{pid}-{i+1}", pid, "finished", "AMB", "ambulatory",
+                  "185349003", "Office Visit", "185349003", "Routine checkup",
+                  start, end))
+    
+    conn.commit()
+    conn.close()
+    
+    print(f"Test database created at {DB_PATH}")
+    print("Contains:")
+    print("  - 3 patients")
+    print("  - 6 conditions")
+    print("  - 6 medications")
+    print("  - ~90 observations (vitals + labs)")
+    print("  - 4 allergies")
+    print("  - 6 immunizations")
+    print("  - 4 procedures")
+    print("  - 15 encounters")
+
+
+if __name__ == "__main__":
+    create_test_database()

evaluation/evaluator.py

@@ -0,0 +1,438 @@
+#!/usr/bin/env python3
+"""
+Evaluator
+
+Compares agent-reported facts against expected values from database.
+Computes accuracy metrics for each comparison.
+"""
+
+from typing import Dict, List, Any, Tuple, Optional
+from dataclasses import dataclass, field
+import math
+
+
+# Tolerances for numerical comparisons
+TOLERANCES = {
+    "vital_value": 0.5,       # BP, heart rate, etc.
+    "lab_value": 0.5,         # Lab results
+    "average": 0.5,           # Computed averages
+    "percentage": 1.0,        # Percentage values
+    "count": 0,               # Counts must be exact
+}
+
+
+@dataclass
+class ComparisonResult:
+    """Result of comparing a single value."""
+    field_name: str
+    expected: Any
+    actual: Any
+    match: bool
+    error_type: Optional[str] = None  # "hallucination", "omission", "mismatch", "tolerance"
+    error_detail: Optional[str] = None
+
+
+@dataclass 
+class CaseEvaluation:
+    """Evaluation result for a single test case."""
+    case_id: str
+    query_type: str
+    success: bool
+    comparisons: List[ComparisonResult] = field(default_factory=list)
+    
+    # Summary stats
+    total_fields: int = 0
+    correct_fields: int = 0
+    hallucinations: int = 0
+    omissions: int = 0
+    mismatches: int = 0
+    
+    def accuracy(self) -> float:
+        if self.total_fields == 0:
+            return 0.0
+        return self.correct_fields / self.total_fields
+
+
+def values_match(expected: Any, actual: Any, tolerance: float = 0) -> bool:
+    """Check if two values match, with optional tolerance for numbers."""
+    if expected is None and actual is None:
+        return True
+    if expected is None or actual is None:
+        return False
+    
+    # Numeric comparison with tolerance
+    if isinstance(expected, (int, float)) and isinstance(actual, (int, float)):
+        return abs(expected - actual) <= tolerance
+    
+    # String comparison (case-insensitive, trimmed)
+    if isinstance(expected, str) and isinstance(actual, str):
+        return expected.lower().strip() == actual.lower().strip()
+    
+    # List comparison (order-independent for certain types)
+    if isinstance(expected, list) and isinstance(actual, list):
+        return set(str(x).lower() for x in expected) == set(str(x).lower() for x in actual)
+    
+    # Default exact comparison
+    return expected == actual
+
+
+def compare_numeric(field_name: str, expected: float, actual: float, 
+                    tolerance: float) -> ComparisonResult:
+    """Compare two numeric values."""
+    if actual is None:
+        return ComparisonResult(
+            field_name=field_name,
+            expected=expected,
+            actual=actual,
+            match=False,
+            error_type="omission",
+            error_detail=f"Expected {expected}, got nothing"
+        )
+    
+    diff = abs(expected - actual)
+    if diff <= tolerance:
+        return ComparisonResult(
+            field_name=field_name,
+            expected=expected,
+            actual=actual,
+            match=True
+        )
+    else:
+        return ComparisonResult(
+            field_name=field_name,
+            expected=expected,
+            actual=actual,
+            match=False,
+            error_type="mismatch" if diff <= tolerance * 3 else "hallucination",
+            error_detail=f"Expected {expected}, got {actual} (diff: {diff:.1f})"
+        )
+
+
+def compare_list_items(field_name: str, expected_items: List[str], 
+                       actual_items: List[str]) -> Tuple[List[ComparisonResult], int, int]:
+    """
+    Compare two lists of items (e.g., medication names).
+    Returns comparisons, hallucination count, omission count.
+    """
+    comparisons = []
+    
+    expected_lower = set(x.lower().strip() for x in expected_items)
+    actual_lower = set(x.lower().strip() for x in actual_items)
+    
+    # Find matches
+    matches = expected_lower & actual_lower
+    
+    # Find omissions (in expected but not actual)
+    omissions = expected_lower - actual_lower
+    
+    # Find hallucinations (in actual but not expected)
+    hallucinations = actual_lower - expected_lower
+    
+    # Record matches
+    for item in matches:
+        comparisons.append(ComparisonResult(
+            field_name=f"{field_name}_item",
+            expected=item,
+            actual=item,
+            match=True
+        ))
+    
+    # Record omissions
+    for item in omissions:
+        comparisons.append(ComparisonResult(
+            field_name=f"{field_name}_item",
+            expected=item,
+            actual=None,
+            match=False,
+            error_type="omission",
+            error_detail=f"Missing: {item}"
+        ))
+    
+    # Record hallucinations
+    for item in hallucinations:
+        comparisons.append(ComparisonResult(
+            field_name=f"{field_name}_item",
+            expected=None,
+            actual=item,
+            match=False,
+            error_type="hallucination",
+            error_detail=f"Not in database: {item}"
+        ))
+    
+    return comparisons, len(hallucinations), len(omissions)
+
+
+def evaluate_vital_trend(expected: Dict, actual_facts: Dict) -> CaseEvaluation:
+    """Evaluate vital trend response against expected values."""
+    evaluation = CaseEvaluation(
+        case_id="",
+        query_type="vital_trend",
+        success=True
+    )
+    
+    if "metrics" not in expected:
+        evaluation.success = False
+        return evaluation
+    
+    for label, expected_metrics in expected["metrics"].items():
+        actual_metrics = actual_facts.get("metrics", {}).get(label, {})
+        
+        # Compare each metric
+        for metric_name in ["min", "max", "avg", "latest", "count"]:
+            if metric_name in expected_metrics:
+                exp_val = expected_metrics[metric_name]
+                act_val = actual_metrics.get(metric_name)
+                
+                tolerance = TOLERANCES["count"] if metric_name == "count" else TOLERANCES["vital_value"]
+                
+                comparison = compare_numeric(
+                    f"{label}_{metric_name}",
+                    exp_val,
+                    act_val,
+                    tolerance
+                )
+                evaluation.comparisons.append(comparison)
+                evaluation.total_fields += 1
+                
+                if comparison.match:
+                    evaluation.correct_fields += 1
+                elif comparison.error_type == "hallucination":
+                    evaluation.hallucinations += 1
+                elif comparison.error_type == "omission":
+                    evaluation.omissions += 1
+                else:
+                    evaluation.mismatches += 1
+        
+        # Compare date range
+        for date_field in ["earliest_date", "latest_date"]:
+            if date_field in expected_metrics:
+                exp_date = expected_metrics[date_field]
+                act_date = actual_metrics.get(date_field)
+                
+                match = values_match(exp_date, act_date)
+                evaluation.comparisons.append(ComparisonResult(
+                    field_name=f"{label}_{date_field}",
+                    expected=exp_date,
+                    actual=act_date,
+                    match=match,
+                    error_type=None if match else "mismatch"
+                ))
+                evaluation.total_fields += 1
+                if match:
+                    evaluation.correct_fields += 1
+                else:
+                    evaluation.mismatches += 1
+    
+    evaluation.success = evaluation.accuracy() >= 0.8  # 80% threshold
+    return evaluation
+
+
+def evaluate_list_query(expected: Dict, actual_facts: Dict, 
+                        items_key: str, names_key: str) -> CaseEvaluation:
+    """
+    Evaluate list-based queries (medications, conditions, allergies, etc.)
+    """
+    evaluation = CaseEvaluation(
+        case_id="",
+        query_type=expected["query_type"],
+        success=True
+    )
+    
+    # Compare count
+    exp_count = expected.get("count", 0)
+    act_count = actual_facts.get("count", 0)
+    
+    count_comparison = compare_numeric("count", exp_count, act_count, TOLERANCES["count"])
+    evaluation.comparisons.append(count_comparison)
+    evaluation.total_fields += 1
+    if count_comparison.match:
+        evaluation.correct_fields += 1
+    
+    # Compare item names
+    exp_names = expected.get(names_key, [])
+    act_names = actual_facts.get(names_key, [])
+    
+    item_comparisons, hallucinations, omissions = compare_list_items(
+        items_key, exp_names, act_names
+    )
+    
+    evaluation.comparisons.extend(item_comparisons)
+    evaluation.total_fields += len(item_comparisons)
+    evaluation.correct_fields += sum(1 for c in item_comparisons if c.match)
+    evaluation.hallucinations += hallucinations
+    evaluation.omissions += omissions
+    
+    evaluation.success = evaluation.accuracy() >= 0.8
+    return evaluation
+
+
+def evaluate_medication_list(expected: Dict, actual_facts: Dict) -> CaseEvaluation:
+    """Evaluate medication list response."""
+    return evaluate_list_query(expected, actual_facts, "medications", "medication_names")
+
+
+def evaluate_condition_list(expected: Dict, actual_facts: Dict) -> CaseEvaluation:
+    """Evaluate condition list response."""
+    return evaluate_list_query(expected, actual_facts, "conditions", "condition_names")
+
+
+def evaluate_allergy_list(expected: Dict, actual_facts: Dict) -> CaseEvaluation:
+    """Evaluate allergy list response."""
+    return evaluate_list_query(expected, actual_facts, "allergies", "substances")
+
+
+def evaluate_immunization_list(expected: Dict, actual_facts: Dict) -> CaseEvaluation:
+    """Evaluate immunization list response."""
+    return evaluate_list_query(expected, actual_facts, "immunizations", "vaccine_names")
+
+
+def evaluate_procedure_list(expected: Dict, actual_facts: Dict) -> CaseEvaluation:
+    """Evaluate procedure list response."""
+    return evaluate_list_query(expected, actual_facts, "procedures", "procedure_names")
+
+
+def evaluate_encounter_list(expected: Dict, actual_facts: Dict) -> CaseEvaluation:
+    """Evaluate encounter list response."""
+    evaluation = CaseEvaluation(
+        case_id="",
+        query_type="encounter_list",
+        success=True
+    )
+    
+    exp_count = expected.get("count", 0)
+    act_count = actual_facts.get("count", 0)
+    
+    # For encounters, we check if count is within the limit
+    limit = expected.get("limit", 5)
+    
+    count_comparison = compare_numeric("count", min(exp_count, limit), act_count, TOLERANCES["count"])
+    evaluation.comparisons.append(count_comparison)
+    evaluation.total_fields += 1
+    if count_comparison.match:
+        evaluation.correct_fields += 1
+    
+    evaluation.success = count_comparison.match
+    return evaluation
+
+
+def evaluate_lab_trend(expected: Dict, actual_facts: Dict) -> CaseEvaluation:
+    """Evaluate lab trend response."""
+    evaluation = CaseEvaluation(
+        case_id="",
+        query_type="lab_trend",
+        success=True
+    )
+    
+    if "metrics" not in expected:
+        evaluation.success = False
+        return evaluation
+    
+    exp_metrics = expected["metrics"]
+    act_metrics = actual_facts.get("metrics", {})
+    
+    for metric_name in ["min", "max", "avg", "latest", "count"]:
+        if metric_name in exp_metrics:
+            exp_val = exp_metrics[metric_name]
+            act_val = act_metrics.get(metric_name)
+            
+            tolerance = TOLERANCES["count"] if metric_name == "count" else TOLERANCES["lab_value"]
+            
+            comparison = compare_numeric(metric_name, exp_val, act_val, tolerance)
+            evaluation.comparisons.append(comparison)
+            evaluation.total_fields += 1
+            
+            if comparison.match:
+                evaluation.correct_fields += 1
+            elif comparison.error_type == "hallucination":
+                evaluation.hallucinations += 1
+            elif comparison.error_type == "omission":
+                evaluation.omissions += 1
+            else:
+                evaluation.mismatches += 1
+    
+    evaluation.success = evaluation.accuracy() >= 0.8
+    return evaluation
+
+
+def evaluate_case(test_case: Dict, expected: Dict, actual_facts: Dict) -> CaseEvaluation:
+    """
+    Evaluate a single test case.
+    
+    Args:
+        test_case: The test case definition
+        expected: Expected values computed from database
+        actual_facts: Facts reported by the agent
+    
+    Returns:
+        CaseEvaluation with detailed comparison results
+    """
+    query_type = test_case["query_type"]
+    
+    evaluators = {
+        "vital_trend": evaluate_vital_trend,
+        "medication_list": evaluate_medication_list,
+        "condition_list": evaluate_condition_list,
+        "allergy_list": evaluate_allergy_list,
+        "immunization_list": evaluate_immunization_list,
+        "procedure_list": evaluate_procedure_list,
+        "encounter_list": evaluate_encounter_list,
+        "lab_trend": evaluate_lab_trend
+    }
+    
+    evaluator = evaluators.get(query_type)
+    if not evaluator:
+        return CaseEvaluation(
+            case_id=test_case["case_id"],
+            query_type=query_type,
+            success=False
+        )
+    
+    evaluation = evaluator(expected, actual_facts)
+    evaluation.case_id = test_case["case_id"]
+    
+    return evaluation
+
+
+if __name__ == "__main__":
+    # Test with sample data
+    expected = {
+        "query_type": "vital_trend",
+        "metrics": {
+            "systolic": {
+                "min": 128.0,
+                "max": 142.0,
+                "avg": 134.8,
+                "count": 5,
+                "earliest_date": "2026-01-22",
+                "latest_date": "2026-01-27"
+            }
+        }
+    }
+    
+    actual = {
+        "metrics": {
+            "systolic": {
+                "min": 128.0,
+                "max": 142.0,
+                "avg": 135.0,  # Slightly off
+                "count": 5,
+                "earliest_date": "2026-01-22",
+                "latest_date": "2026-01-27"
+            }
+        }
+    }
+    
+    test_case = {"case_id": "test_1", "query_type": "vital_trend"}
+    
+    result = evaluate_case(test_case, expected, actual)
+    
+    print(f"Case: {result.case_id}")
+    print(f"Success: {result.success}")
+    print(f"Accuracy: {result.accuracy():.1%}")
+    print(f"Fields: {result.correct_fields}/{result.total_fields}")
+    print(f"Hallucinations: {result.hallucinations}")
+    print(f"Omissions: {result.omissions}")
+    print("\nComparisons:")
+    for c in result.comparisons:
+        status = "✓" if c.match else "✗"
+        print(f"  {status} {c.field_name}: expected={c.expected}, actual={c.actual}")

evaluation/expected_values.py

@@ -0,0 +1,386 @@
+#!/usr/bin/env python3
+"""
+Expected Values Calculator
+
+Computes ground truth values directly from the database for each test case type.
+These are the values we expect the LLM to report.
+"""
+
+import sqlite3
+from datetime import datetime, timedelta
+from typing import Dict, List, Any, Optional
+import os
+import statistics
+
+DB_PATH = os.getenv("DB_PATH", "data/fhir.db")
+
+
+def get_db():
+    """Get database connection."""
+    conn = sqlite3.connect(DB_PATH)
+    conn.row_factory = sqlite3.Row
+    return conn
+
+
+def compute_vital_trend_expected(patient_id: str, vital_type: str, codes: List[str], 
+                                  labels: List[str], days: int = 30) -> Dict[str, Any]:
+    """
+    Compute expected values for vital trend queries.
+    
+    Returns expected facts like min, max, avg, count, date range.
+    """
+    conn = get_db()
+    try:
+        cutoff_date = (datetime.now() - timedelta(days=days)).strftime("%Y-%m-%d")
+        
+        result = {
+            "query_type": "vital_trend",
+            "vital_type": vital_type,
+            "days": days,
+            "metrics": {}
+        }
+        
+        for code, label in zip(codes, labels):
+            cursor = conn.execute("""
+                SELECT value_quantity, effective_date
+                FROM observations
+                WHERE patient_id = ? AND code = ? AND effective_date >= ?
+                ORDER BY effective_date ASC
+            """, (patient_id, code, cutoff_date))
+            
+            rows = cursor.fetchall()
+            values = [r["value_quantity"] for r in rows if r["value_quantity"] is not None]
+            dates = [r["effective_date"][:10] for r in rows]
+            
+            if values:
+                result["metrics"][label] = {
+                    "min": round(min(values), 1),
+                    "max": round(max(values), 1),
+                    "avg": round(statistics.mean(values), 1),
+                    "count": len(values),
+                    "latest": round(values[-1], 1),
+                    "earliest_date": dates[0] if dates else None,
+                    "latest_date": dates[-1] if dates else None,
+                    "all_values": [round(v, 1) for v in values],
+                    "all_dates": dates
+                }
+        
+        return result
+    finally:
+        conn.close()
+
+
+def compute_medication_expected(patient_id: str, status: Optional[str] = None) -> Dict[str, Any]:
+    """
+    Compute expected values for medication queries.
+    
+    Returns list of medications with their details.
+    """
+    conn = get_db()
+    try:
+        if status:
+            cursor = conn.execute("""
+                SELECT code, display, status, start_date
+                FROM medications
+                WHERE patient_id = ? AND status = ?
+                ORDER BY start_date DESC
+            """, (patient_id, status))
+        else:
+            cursor = conn.execute("""
+                SELECT code, display, status, start_date
+                FROM medications
+                WHERE patient_id = ?
+                ORDER BY start_date DESC
+            """, (patient_id,))
+        
+        medications = []
+        for row in cursor.fetchall():
+            medications.append({
+                "code": row["code"],
+                "display": row["display"],
+                "status": row["status"],
+                "start_date": row["start_date"][:10] if row["start_date"] else None
+            })
+        
+        return {
+            "query_type": "medication_list",
+            "status_filter": status,
+            "count": len(medications),
+            "medications": medications,
+            "medication_names": [m["display"] for m in medications]
+        }
+    finally:
+        conn.close()
+
+
+def compute_condition_expected(patient_id: str) -> Dict[str, Any]:
+    """
+    Compute expected values for condition queries.
+    """
+    conn = get_db()
+    try:
+        cursor = conn.execute("""
+            SELECT code, display, clinical_status, onset_date
+            FROM conditions
+            WHERE patient_id = ?
+            ORDER BY onset_date DESC
+        """, (patient_id,))
+        
+        conditions = []
+        for row in cursor.fetchall():
+            conditions.append({
+                "code": row["code"],
+                "display": row["display"],
+                "clinical_status": row["clinical_status"],
+                "onset_date": row["onset_date"][:10] if row["onset_date"] else None
+            })
+        
+        return {
+            "query_type": "condition_list",
+            "count": len(conditions),
+            "conditions": conditions,
+            "condition_names": [c["display"] for c in conditions]
+        }
+    finally:
+        conn.close()
+
+
+def compute_allergy_expected(patient_id: str) -> Dict[str, Any]:
+    """
+    Compute expected values for allergy queries.
+    """
+    conn = get_db()
+    try:
+        cursor = conn.execute("""
+            SELECT substance, reaction_display, criticality, category
+            FROM allergies
+            WHERE patient_id = ?
+        """, (patient_id,))
+        
+        allergies = []
+        for row in cursor.fetchall():
+            allergies.append({
+                "substance": row["substance"],
+                "reaction": row["reaction_display"],
+                "criticality": row["criticality"],
+                "category": row["category"]
+            })
+        
+        return {
+            "query_type": "allergy_list",
+            "count": len(allergies),
+            "allergies": allergies,
+            "substances": [a["substance"] for a in allergies]
+        }
+    finally:
+        conn.close()
+
+
+def compute_immunization_expected(patient_id: str) -> Dict[str, Any]:
+    """
+    Compute expected values for immunization queries.
+    """
+    conn = get_db()
+    try:
+        cursor = conn.execute("""
+            SELECT vaccine_code, vaccine_display, status, occurrence_date
+            FROM immunizations
+            WHERE patient_id = ?
+            ORDER BY occurrence_date DESC
+        """, (patient_id,))
+        
+        immunizations = []
+        for row in cursor.fetchall():
+            immunizations.append({
+                "vaccine_code": row["vaccine_code"],
+                "vaccine_display": row["vaccine_display"],
+                "status": row["status"],
+                "occurrence_date": row["occurrence_date"][:10] if row["occurrence_date"] else None
+            })
+        
+        return {
+            "query_type": "immunization_list",
+            "count": len(immunizations),
+            "immunizations": immunizations,
+            "vaccine_names": [i["vaccine_display"] for i in immunizations]
+        }
+    finally:
+        conn.close()
+
+
+def compute_procedure_expected(patient_id: str) -> Dict[str, Any]:
+    """
+    Compute expected values for procedure queries.
+    """
+    conn = get_db()
+    try:
+        cursor = conn.execute("""
+            SELECT code, display, status, performed_date
+            FROM procedures
+            WHERE patient_id = ?
+            ORDER BY performed_date DESC
+        """, (patient_id,))
+        
+        procedures = []
+        for row in cursor.fetchall():
+            procedures.append({
+                "code": row["code"],
+                "display": row["display"],
+                "status": row["status"],
+                "performed_date": row["performed_date"][:10] if row["performed_date"] else None
+            })
+        
+        return {
+            "query_type": "procedure_list",
+            "count": len(procedures),
+            "procedures": procedures,
+            "procedure_names": [p["display"] for p in procedures]
+        }
+    finally:
+        conn.close()
+
+
+def compute_encounter_expected(patient_id: str, limit: int = 5) -> Dict[str, Any]:
+    """
+    Compute expected values for encounter queries.
+    """
+    conn = get_db()
+    try:
+        cursor = conn.execute("""
+            SELECT type_display, reason_display, period_start, period_end, class_display
+            FROM encounters
+            WHERE patient_id = ?
+            ORDER BY period_start DESC
+            LIMIT ?
+        """, (patient_id, limit))
+        
+        encounters = []
+        for row in cursor.fetchall():
+            encounters.append({
+                "type": row["type_display"],
+                "reason": row["reason_display"],
+                "class": row["class_display"],
+                "start_date": row["period_start"][:10] if row["period_start"] else None,
+                "end_date": row["period_end"][:10] if row["period_end"] else None
+            })
+        
+        return {
+            "query_type": "encounter_list",
+            "count": len(encounters),
+            "limit": limit,
+            "encounters": encounters
+        }
+    finally:
+        conn.close()
+
+
+def compute_lab_trend_expected(patient_id: str, lab_type: str, code: str, 
+                                periods: int = 4) -> Dict[str, Any]:
+    """
+    Compute expected values for lab trend queries.
+    """
+    conn = get_db()
+    try:
+        cursor = conn.execute("""
+            SELECT value_quantity, effective_date, unit
+            FROM observations
+            WHERE patient_id = ? AND code = ?
+            ORDER BY effective_date DESC
+            LIMIT ?
+        """, (patient_id, code, periods))
+        
+        rows = cursor.fetchall()
+        values = [r["value_quantity"] for r in rows if r["value_quantity"] is not None]
+        dates = [r["effective_date"][:10] for r in rows]
+        unit = rows[0]["unit"] if rows else None
+        
+        result = {
+            "query_type": "lab_trend",
+            "lab_type": lab_type,
+            "code": code,
+            "unit": unit,
+            "count": len(values)
+        }
+        
+        if values:
+            result["metrics"] = {
+                "min": round(min(values), 1),
+                "max": round(max(values), 1),
+                "avg": round(statistics.mean(values), 1),
+                "latest": round(values[0], 1),  # Most recent
+                "latest_date": dates[0] if dates else None,
+                "all_values": [round(v, 1) for v in values],
+                "all_dates": dates
+            }
+        
+        return result
+    finally:
+        conn.close()
+
+
+def compute_expected_values(test_case: Dict) -> Dict[str, Any]:
+    """
+    Compute expected values for any test case type.
+    Routes to the appropriate computation function.
+    """
+    query_type = test_case["query_type"]
+    patient_id = test_case["patient_id"]
+    params = test_case.get("parameters", {})
+    
+    if query_type == "vital_trend":
+        return compute_vital_trend_expected(
+            patient_id,
+            params["vital_type"],
+            params["codes"],
+            params["labels"],
+            params.get("days", 30)
+        )
+    
+    elif query_type == "medication_list":
+        return compute_medication_expected(patient_id, params.get("status"))
+    
+    elif query_type == "condition_list":
+        return compute_condition_expected(patient_id)
+    
+    elif query_type == "allergy_list":
+        return compute_allergy_expected(patient_id)
+    
+    elif query_type == "immunization_list":
+        return compute_immunization_expected(patient_id)
+    
+    elif query_type == "procedure_list":
+        return compute_procedure_expected(patient_id)
+    
+    elif query_type == "encounter_list":
+        return compute_encounter_expected(patient_id, params.get("limit", 5))
+    
+    elif query_type == "lab_trend":
+        return compute_lab_trend_expected(
+            patient_id,
+            params["lab_type"],
+            params["code"],
+            params.get("periods", 4)
+        )
+    
+    else:
+        return {"error": f"Unknown query type: {query_type}"}
+
+
+if __name__ == "__main__":
+    # Test with a sample case
+    from test_generator import generate_all_test_cases
+    import json
+    
+    print("Generating test cases...")
+    cases = generate_all_test_cases(num_patients=1)
+    
+    print(f"\nComputing expected values for {len(cases)} test cases...")
+    
+    for case in cases[:3]:  # Show first 3
+        print(f"\n{'='*60}")
+        print(f"Case: {case['case_id']}")
+        print(f"Query: {case['query']}")
+        
+        expected = compute_expected_values(case)
+        print(f"Expected values:")
+        print(json.dumps(expected, indent=2, default=str))

evaluation/facts_schema.py

@@ -0,0 +1,232 @@
+#!/usr/bin/env python3
+"""
+Facts Schema
+
+Defines the structured facts format that the agent should return
+alongside its text responses. These facts are used for evaluation.
+"""
+
+from typing import Dict, List, Any, Optional
+from dataclasses import dataclass, asdict
+import json
+
+
+@dataclass
+class VitalTrendFacts:
+    """Structured facts for vital sign trend queries."""
+    vital_type: str
+    days: int
+    metrics: Dict[str, Dict[str, Any]]  # {label: {min, max, avg, count, dates...}}
+    
+    def to_dict(self) -> Dict:
+        return asdict(self)
+
+
+@dataclass
+class MedicationFacts:
+    """Structured facts for medication queries."""
+    status_filter: Optional[str]
+    count: int
+    medication_names: List[str]
+    
+    def to_dict(self) -> Dict:
+        return asdict(self)
+
+
+@dataclass
+class ConditionFacts:
+    """Structured facts for condition queries."""
+    count: int
+    condition_names: List[str]
+    
+    def to_dict(self) -> Dict:
+        return asdict(self)
+
+
+@dataclass
+class AllergyFacts:
+    """Structured facts for allergy queries."""
+    count: int
+    substances: List[str]
+    
+    def to_dict(self) -> Dict:
+        return asdict(self)
+
+
+@dataclass
+class ImmunizationFacts:
+    """Structured facts for immunization queries."""
+    count: int
+    vaccine_names: List[str]
+    
+    def to_dict(self) -> Dict:
+        return asdict(self)
+
+
+@dataclass
+class ProcedureFacts:
+    """Structured facts for procedure queries."""
+    count: int
+    procedure_names: List[str]
+    
+    def to_dict(self) -> Dict:
+        return asdict(self)
+
+
+@dataclass
+class EncounterFacts:
+    """Structured facts for encounter queries."""
+    count: int
+    limit: int
+    
+    def to_dict(self) -> Dict:
+        return asdict(self)
+
+
+@dataclass
+class LabTrendFacts:
+    """Structured facts for lab trend queries."""
+    lab_type: str
+    code: str
+    unit: Optional[str]
+    count: int
+    metrics: Dict[str, Any]  # {min, max, avg, latest, dates...}
+    
+    def to_dict(self) -> Dict:
+        return asdict(self)
+
+
+def extract_vital_facts_from_tool_result(tool_result: Dict) -> Optional[VitalTrendFacts]:
+    """
+    Extract structured facts from vital chart tool result.
+    The tool already returns structured JSON - we just reshape it.
+    """
+    if "error" in tool_result:
+        return None
+    
+    chart_type = tool_result.get("chart_type", "")
+    if chart_type not in ["line", "line_dual"]:
+        return None
+    
+    metrics = {}
+    
+    for dataset in tool_result.get("datasets", []):
+        label = dataset.get("label", "unknown").lower().replace(" ", "_")
+        data_points = dataset.get("data", [])
+        
+        if not data_points:
+            continue
+        
+        values = [p["value"] for p in data_points if p.get("value") is not None]
+        dates = [p["date"] for p in data_points if p.get("date")]
+        
+        if values:
+            import statistics
+            metrics[label] = {
+                "min": round(min(values), 1),
+                "max": round(max(values), 1),
+                "avg": round(statistics.mean(values), 1),
+                "count": len(values),
+                "latest": round(values[-1], 1) if values else None,
+                "earliest_date": dates[0] if dates else None,
+                "latest_date": dates[-1] if dates else None
+            }
+    
+    return VitalTrendFacts(
+        vital_type=tool_result.get("title", "").lower().replace(" ", "_"),
+        days=30,  # Default, could be extracted from title
+        metrics=metrics
+    )
+
+
+def extract_lab_facts_from_tool_result(tool_result: Dict) -> Optional[LabTrendFacts]:
+    """Extract structured facts from lab chart tool result."""
+    if "error" in tool_result:
+        return None
+    
+    datasets = tool_result.get("datasets", [])
+    if not datasets:
+        return None
+    
+    # Get first dataset
+    dataset = datasets[0]
+    data_points = dataset.get("data", [])
+    
+    if not data_points:
+        return None
+    
+    values = [p["value"] for p in data_points if p.get("value") is not None]
+    dates = [p["date"] for p in data_points if p.get("date")]
+    
+    metrics = {}
+    if values:
+        import statistics
+        metrics = {
+            "min": round(min(values), 1),
+            "max": round(max(values), 1),
+            "avg": round(statistics.mean(values), 1),
+            "latest": round(values[-1], 1),
+            "latest_date": dates[-1] if dates else None
+        }
+    
+    return LabTrendFacts(
+        lab_type=dataset.get("label", "unknown").lower(),
+        code="",  # Not in tool result
+        unit=tool_result.get("unit"),
+        count=len(values),
+        metrics=metrics
+    )
+
+
+def extract_medication_facts(medications: List[Dict], status_filter: Optional[str] = None) -> MedicationFacts:
+    """Extract structured facts from medication list."""
+    names = [m.get("display", "") for m in medications]
+    return MedicationFacts(
+        status_filter=status_filter,
+        count=len(medications),
+        medication_names=names
+    )
+
+
+def extract_condition_facts(conditions: List[Dict]) -> ConditionFacts:
+    """Extract structured facts from condition list."""
+    names = [c.get("display", "") for c in conditions]
+    return ConditionFacts(
+        count=len(conditions),
+        condition_names=names
+    )
+
+
+def extract_allergy_facts(allergies: List[Dict]) -> AllergyFacts:
+    """Extract structured facts from allergy list."""
+    substances = [a.get("substance", "") for a in allergies]
+    return AllergyFacts(
+        count=len(allergies),
+        substances=substances
+    )
+
+
+def extract_immunization_facts(immunizations: List[Dict]) -> ImmunizationFacts:
+    """Extract structured facts from immunization list."""
+    names = [i.get("vaccine_display", "") for i in immunizations]
+    return ImmunizationFacts(
+        count=len(immunizations),
+        vaccine_names=names
+    )
+
+
+def extract_procedure_facts(procedures: List[Dict]) -> ProcedureFacts:
+    """Extract structured facts from procedure list."""
+    names = [p.get("display", "") for p in procedures]
+    return ProcedureFacts(
+        count=len(procedures),
+        procedure_names=names
+    )
+
+
+def extract_encounter_facts(encounters: List[Dict], limit: int = 5) -> EncounterFacts:
+    """Extract structured facts from encounter list."""
+    return EncounterFacts(
+        count=len(encounters),
+        limit=limit
+    )

evaluation/metrics.py

@@ -0,0 +1,265 @@
+#!/usr/bin/env python3
+"""
+Metrics Calculator
+
+Aggregates evaluation results across multiple test cases
+and computes summary statistics.
+"""
+
+from typing import Dict, List, Any
+from dataclasses import dataclass, field
+from datetime import datetime
+import json
+
+from .evaluator import CaseEvaluation
+
+
+@dataclass
+class EvaluationMetrics:
+    """Aggregated metrics across all test cases."""
+    
+    # Overall
+    total_cases: int = 0
+    successful_cases: int = 0
+    failed_cases: int = 0
+    
+    # Field-level
+    total_fields: int = 0
+    correct_fields: int = 0
+    total_hallucinations: int = 0
+    total_omissions: int = 0
+    total_mismatches: int = 0
+    
+    # By query type
+    by_query_type: Dict[str, Dict[str, Any]] = field(default_factory=dict)
+    
+    # Detailed results
+    case_results: List[Dict] = field(default_factory=list)
+    
+    def success_rate(self) -> float:
+        if self.total_cases == 0:
+            return 0.0
+        return self.successful_cases / self.total_cases
+    
+    def field_accuracy(self) -> float:
+        if self.total_fields == 0:
+            return 0.0
+        return self.correct_fields / self.total_fields
+    
+    def hallucination_rate(self) -> float:
+        if self.total_fields == 0:
+            return 0.0
+        return self.total_hallucinations / self.total_fields
+    
+    def omission_rate(self) -> float:
+        if self.total_fields == 0:
+            return 0.0
+        return self.total_omissions / self.total_fields
+    
+    def to_dict(self) -> Dict:
+        return {
+            "summary": {
+                "total_cases": self.total_cases,
+                "successful_cases": self.successful_cases,
+                "failed_cases": self.failed_cases,
+                "success_rate": f"{self.success_rate():.1%}",
+                "field_accuracy": f"{self.field_accuracy():.1%}",
+                "hallucination_rate": f"{self.hallucination_rate():.1%}",
+                "omission_rate": f"{self.omission_rate():.1%}"
+            },
+            "field_level": {
+                "total_fields": self.total_fields,
+                "correct_fields": self.correct_fields,
+                "hallucinations": self.total_hallucinations,
+                "omissions": self.total_omissions,
+                "mismatches": self.total_mismatches
+            },
+            "by_query_type": self.by_query_type,
+            "case_results": self.case_results
+        }
+
+
+def aggregate_metrics(evaluations: List[CaseEvaluation]) -> EvaluationMetrics:
+    """
+    Aggregate metrics from multiple case evaluations.
+    """
+    metrics = EvaluationMetrics()
+    
+    for eval_result in evaluations:
+        metrics.total_cases += 1
+        
+        if eval_result.success:
+            metrics.successful_cases += 1
+        else:
+            metrics.failed_cases += 1
+        
+        # Field-level aggregation
+        metrics.total_fields += eval_result.total_fields
+        metrics.correct_fields += eval_result.correct_fields
+        metrics.total_hallucinations += eval_result.hallucinations
+        metrics.total_omissions += eval_result.omissions
+        metrics.total_mismatches += eval_result.mismatches
+        
+        # By query type
+        qtype = eval_result.query_type
+        if qtype not in metrics.by_query_type:
+            metrics.by_query_type[qtype] = {
+                "total": 0,
+                "successful": 0,
+                "failed": 0,
+                "total_fields": 0,
+                "correct_fields": 0,
+                "hallucinations": 0,
+                "omissions": 0
+            }
+        
+        metrics.by_query_type[qtype]["total"] += 1
+        if eval_result.success:
+            metrics.by_query_type[qtype]["successful"] += 1
+        else:
+            metrics.by_query_type[qtype]["failed"] += 1
+        metrics.by_query_type[qtype]["total_fields"] += eval_result.total_fields
+        metrics.by_query_type[qtype]["correct_fields"] += eval_result.correct_fields
+        metrics.by_query_type[qtype]["hallucinations"] += eval_result.hallucinations
+        metrics.by_query_type[qtype]["omissions"] += eval_result.omissions
+        
+        # Store case result
+        metrics.case_results.append({
+            "case_id": eval_result.case_id,
+            "query_type": eval_result.query_type,
+            "success": eval_result.success,
+            "accuracy": eval_result.accuracy(),
+            "fields": f"{eval_result.correct_fields}/{eval_result.total_fields}",
+            "hallucinations": eval_result.hallucinations,
+            "omissions": eval_result.omissions
+        })
+    
+    # Calculate per-type success rates
+    for qtype, stats in metrics.by_query_type.items():
+        if stats["total"] > 0:
+            stats["success_rate"] = f"{stats['successful'] / stats['total']:.1%}"
+        if stats["total_fields"] > 0:
+            stats["field_accuracy"] = f"{stats['correct_fields'] / stats['total_fields']:.1%}"
+    
+    return metrics
+
+
+def format_report(metrics: EvaluationMetrics) -> str:
+    """Format metrics as a human-readable report."""
+    lines = []
+    
+    lines.append("=" * 60)
+    lines.append("PRE-VISIT SUMMARY EVALUATION REPORT")
+    lines.append(f"Generated: {datetime.now().strftime('%Y-%m-%d %H:%M:%S')}")
+    lines.append("=" * 60)
+    
+    lines.append("")
+    lines.append("OVERALL RESULTS")
+    lines.append("-" * 40)
+    lines.append(f"Total Test Cases:     {metrics.total_cases}")
+    lines.append(f"Successful:           {metrics.successful_cases}")
+    lines.append(f"Failed:               {metrics.failed_cases}")
+    lines.append(f"Success Rate:         {metrics.success_rate():.1%}")
+    lines.append("")
+    lines.append(f"Total Fields Checked: {metrics.total_fields}")
+    lines.append(f"Correct Fields:       {metrics.correct_fields}")
+    lines.append(f"Field Accuracy:       {metrics.field_accuracy():.1%}")
+    lines.append("")
+    lines.append(f"Hallucinations:       {metrics.total_hallucinations} ({metrics.hallucination_rate():.1%})")
+    lines.append(f"Omissions:            {metrics.total_omissions} ({metrics.omission_rate():.1%})")
+    lines.append(f"Mismatches:           {metrics.total_mismatches}")
+    
+    lines.append("")
+    lines.append("BY QUERY TYPE")
+    lines.append("-" * 40)
+    
+    # Sort by total count
+    sorted_types = sorted(metrics.by_query_type.items(), 
+                         key=lambda x: x[1]["total"], reverse=True)
+    
+    # Header
+    lines.append(f"{'Query Type':<25} {'Success':<12} {'Accuracy':<12} {'Hall.':<8}")
+    lines.append("-" * 60)
+    
+    for qtype, stats in sorted_types:
+        success_rate = stats.get("success_rate", "N/A")
+        field_acc = stats.get("field_accuracy", "N/A")
+        lines.append(f"{qtype:<25} {success_rate:<12} {field_acc:<12} {stats['hallucinations']:<8}")
+    
+    # Failed cases detail
+    failed_cases = [c for c in metrics.case_results if not c["success"]]
+    if failed_cases:
+        lines.append("")
+        lines.append("FAILED CASES")
+        lines.append("-" * 40)
+        for case in failed_cases[:10]:  # Show first 10
+            lines.append(f"  {case['case_id']}")
+            lines.append(f"    Type: {case['query_type']}, Accuracy: {case['accuracy']:.1%}")
+            lines.append(f"    Hallucinations: {case['hallucinations']}, Omissions: {case['omissions']}")
+    
+    lines.append("")
+    lines.append("=" * 60)
+    
+    return "\n".join(lines)
+
+
+def save_report(metrics: EvaluationMetrics, output_dir: str = "."):
+    """Save evaluation report to files."""
+    import os
+    
+    os.makedirs(output_dir, exist_ok=True)
+    timestamp = datetime.now().strftime("%Y%m%d_%H%M%S")
+    
+    # Save text report
+    text_path = os.path.join(output_dir, f"eval_report_{timestamp}.txt")
+    with open(text_path, "w") as f:
+        f.write(format_report(metrics))
+    
+    # Save JSON report
+    json_path = os.path.join(output_dir, f"eval_report_{timestamp}.json")
+    with open(json_path, "w") as f:
+        json.dump(metrics.to_dict(), f, indent=2, default=str)
+    
+    return text_path, json_path
+
+
+if __name__ == "__main__":
+    # Test with sample evaluations
+    from evaluator import CaseEvaluation, ComparisonResult
+    
+    # Create some sample evaluations
+    evaluations = [
+        CaseEvaluation(
+            case_id="patient1_vital_bp",
+            query_type="vital_trend",
+            success=True,
+            total_fields=10,
+            correct_fields=9,
+            hallucinations=0,
+            omissions=1,
+            mismatches=0
+        ),
+        CaseEvaluation(
+            case_id="patient1_meds",
+            query_type="medication_list",
+            success=True,
+            total_fields=5,
+            correct_fields=5,
+            hallucinations=0,
+            omissions=0,
+            mismatches=0
+        ),
+        CaseEvaluation(
+            case_id="patient1_conditions",
+            query_type="condition_list",
+            success=False,
+            total_fields=8,
+            correct_fields=5,
+            hallucinations=2,
+            omissions=1,
+            mismatches=0
+        ),
+    ]
+    
+    metrics = aggregate_metrics(evaluations)
+    print(format_report(metrics))

evaluation/reports/eval_report_20260127_174121.json

@@ -0,0 +1,380 @@
+{
+  "summary": {
+    "total_cases": 30,
+    "successful_cases": 30,
+    "failed_cases": 0,
+    "success_rate": "100.0%",
+    "field_accuracy": "100.0%",
+    "hallucination_rate": "0.0%",
+    "omission_rate": "0.0%"
+  },
+  "field_level": {
+    "total_fields": 128,
+    "correct_fields": 128,
+    "hallucinations": 0,
+    "omissions": 0,
+    "mismatches": 0
+  },
+  "by_query_type": {
+    "vital_trend": {
+      "total": 6,
+      "successful": 6,
+      "failed": 0,
+      "total_fields": 63,
+      "correct_fields": 63,
+      "hallucinations": 0,
+      "omissions": 0,
+      "success_rate": "100.0%",
+      "field_accuracy": "100.0%"
+    },
+    "medication_list": {
+      "total": 6,
+      "successful": 6,
+      "failed": 0,
+      "total_fields": 18,
+      "correct_fields": 18,
+      "hallucinations": 0,
+      "omissions": 0,
+      "success_rate": "100.0%",
+      "field_accuracy": "100.0%"
+    },
+    "condition_list": {
+      "total": 3,
+      "successful": 3,
+      "failed": 0,
+      "total_fields": 9,
+      "correct_fields": 9,
+      "hallucinations": 0,
+      "omissions": 0,
+      "success_rate": "100.0%",
+      "field_accuracy": "100.0%"
+    },
+    "allergy_list": {
+      "total": 3,
+      "successful": 3,
+      "failed": 0,
+      "total_fields": 7,
+      "correct_fields": 7,
+      "hallucinations": 0,
+      "omissions": 0,
+      "success_rate": "100.0%",
+      "field_accuracy": "100.0%"
+    },
+    "immunization_list": {
+      "total": 3,
+      "successful": 3,
+      "failed": 0,
+      "total_fields": 9,
+      "correct_fields": 9,
+      "hallucinations": 0,
+      "omissions": 0,
+      "success_rate": "100.0%",
+      "field_accuracy": "100.0%"
+    },
+    "procedure_list": {
+      "total": 3,
+      "successful": 3,
+      "failed": 0,
+      "total_fields": 7,
+      "correct_fields": 7,
+      "hallucinations": 0,
+      "omissions": 0,
+      "success_rate": "100.0%",
+      "field_accuracy": "100.0%"
+    },
+    "encounter_list": {
+      "total": 3,
+      "successful": 3,
+      "failed": 0,
+      "total_fields": 3,
+      "correct_fields": 3,
+      "hallucinations": 0,
+      "omissions": 0,
+      "success_rate": "100.0%",
+      "field_accuracy": "100.0%"
+    },
+    "lab_trend": {
+      "total": 3,
+      "successful": 3,
+      "failed": 0,
+      "total_fields": 12,
+      "correct_fields": 12,
+      "hallucinations": 0,
+      "omissions": 0,
+      "success_rate": "100.0%",
+      "field_accuracy": "100.0%"
+    }
+  },
+  "case_results": [
+    {
+      "case_id": "patient-001_vital_blood_pressure",
+      "query_type": "vital_trend",
+      "success": true,
+      "accuracy": 1.0,
+      "fields": "14/14",
+      "hallucinations": 0,
+      "omissions": 0
+    },
+    {
+      "case_id": "patient-001_vital_heart_rate",
+      "query_type": "vital_trend",
+      "success": true,
+      "accuracy": 1.0,
+      "fields": "7/7",
+      "hallucinations": 0,
+      "omissions": 0
+    },
+    {
+      "case_id": "patient-001_meds_all",
+      "query_type": "medication_list",
+      "success": true,
+      "accuracy": 1.0,
+      "fields": "3/3",
+      "hallucinations": 0,
+      "omissions": 0
+    },
+    {
+      "case_id": "patient-001_meds_active",
+      "query_type": "medication_list",
+      "success": true,
+      "accuracy": 1.0,
+      "fields": "3/3",
+      "hallucinations": 0,
+      "omissions": 0
+    },
+    {
+      "case_id": "patient-001_conditions",
+      "query_type": "condition_list",
+      "success": true,
+      "accuracy": 1.0,
+      "fields": "3/3",
+      "hallucinations": 0,
+      "omissions": 0
+    },
+    {
+      "case_id": "patient-001_allergies",
+      "query_type": "allergy_list",
+      "success": true,
+      "accuracy": 1.0,
+      "fields": "2/2",
+      "hallucinations": 0,
+      "omissions": 0
+    },
+    {
+      "case_id": "patient-001_immunizations",
+      "query_type": "immunization_list",
+      "success": true,
+      "accuracy": 1.0,
+      "fields": "3/3",
+      "hallucinations": 0,
+      "omissions": 0
+    },
+    {
+      "case_id": "patient-001_procedures",
+      "query_type": "procedure_list",
+      "success": true,
+      "accuracy": 1.0,
+      "fields": "2/2",
+      "hallucinations": 0,
+      "omissions": 0
+    },
+    {
+      "case_id": "patient-001_encounters",
+      "query_type": "encounter_list",
+      "success": true,
+      "accuracy": 1.0,
+      "fields": "1/1",
+      "hallucinations": 0,
+      "omissions": 0
+    },
+    {
+      "case_id": "patient-001_lab_a1c",
+      "query_type": "lab_trend",
+      "success": true,
+      "accuracy": 1.0,
+      "fields": "4/4",
+      "hallucinations": 0,
+      "omissions": 0
+    },
+    {
+      "case_id": "patient-002_vital_blood_pressure",
+      "query_type": "vital_trend",
+      "success": true,
+      "accuracy": 1.0,
+      "fields": "14/14",
+      "hallucinations": 0,
+      "omissions": 0
+    },
+    {
+      "case_id": "patient-002_vital_heart_rate",
+      "query_type": "vital_trend",
+      "success": true,
+      "accuracy": 1.0,
+      "fields": "7/7",
+      "hallucinations": 0,
+      "omissions": 0
+    },
+    {
+      "case_id": "patient-002_meds_all",
+      "query_type": "medication_list",
+      "success": true,
+      "accuracy": 1.0,
+      "fields": "3/3",
+      "hallucinations": 0,
+      "omissions": 0
+    },
+    {
+      "case_id": "patient-002_meds_active",
+      "query_type": "medication_list",
+      "success": true,
+      "accuracy": 1.0,
+      "fields": "3/3",
+      "hallucinations": 0,
+      "omissions": 0
+    },
+    {
+      "case_id": "patient-002_conditions",
+      "query_type": "condition_list",
+      "success": true,
+      "accuracy": 1.0,
+      "fields": "3/3",
+      "hallucinations": 0,
+      "omissions": 0
+    },
+    {
+      "case_id": "patient-002_allergies",
+      "query_type": "allergy_list",
+      "success": true,
+      "accuracy": 1.0,
+      "fields": "3/3",
+      "hallucinations": 0,
+      "omissions": 0
+    },
+    {
+      "case_id": "patient-002_immunizations",
+      "query_type": "immunization_list",
+      "success": true,
+      "accuracy": 1.0,
+      "fields": "3/3",
+      "hallucinations": 0,
+      "omissions": 0
+    },
+    {
+      "case_id": "patient-002_procedures",
+      "query_type": "procedure_list",
+      "success": true,
+      "accuracy": 1.0,
+      "fields": "2/2",
+      "hallucinations": 0,
+      "omissions": 0
+    },
+    {
+      "case_id": "patient-002_encounters",
+      "query_type": "encounter_list",
+      "success": true,
+      "accuracy": 1.0,
+      "fields": "1/1",
+      "hallucinations": 0,
+      "omissions": 0
+    },
+    {
+      "case_id": "patient-002_lab_a1c",
+      "query_type": "lab_trend",
+      "success": true,
+      "accuracy": 1.0,
+      "fields": "4/4",
+      "hallucinations": 0,
+      "omissions": 0
+    },
+    {
+      "case_id": "patient-003_vital_blood_pressure",
+      "query_type": "vital_trend",
+      "success": true,
+      "accuracy": 1.0,
+      "fields": "14/14",
+      "hallucinations": 0,
+      "omissions": 0
+    },
+    {
+      "case_id": "patient-003_vital_heart_rate",
+      "query_type": "vital_trend",
+      "success": true,
+      "accuracy": 1.0,
+      "fields": "7/7",
+      "hallucinations": 0,
+      "omissions": 0
+    },
+    {
+      "case_id": "patient-003_meds_all",
+      "query_type": "medication_list",
+      "success": true,
+      "accuracy": 1.0,
+      "fields": "3/3",
+      "hallucinations": 0,
+      "omissions": 0
+    },
+    {
+      "case_id": "patient-003_meds_active",
+      "query_type": "medication_list",
+      "success": true,
+      "accuracy": 1.0,
+      "fields": "3/3",
+      "hallucinations": 0,
+      "omissions": 0
+    },
+    {
+      "case_id": "patient-003_conditions",
+      "query_type": "condition_list",
+      "success": true,
+      "accuracy": 1.0,
+      "fields": "3/3",
+      "hallucinations": 0,
+      "omissions": 0
+    },
+    {
+      "case_id": "patient-003_allergies",
+      "query_type": "allergy_list",
+      "success": true,
+      "accuracy": 1.0,
+      "fields": "2/2",
+      "hallucinations": 0,
+      "omissions": 0
+    },
+    {
+      "case_id": "patient-003_immunizations",
+      "query_type": "immunization_list",
+      "success": true,
+      "accuracy": 1.0,
+      "fields": "3/3",
+      "hallucinations": 0,
+      "omissions": 0
+    },
+    {
+      "case_id": "patient-003_procedures",
+      "query_type": "procedure_list",
+      "success": true,
+      "accuracy": 1.0,
+      "fields": "3/3",
+      "hallucinations": 0,
+      "omissions": 0
+    },
+    {
+      "case_id": "patient-003_encounters",
+      "query_type": "encounter_list",
+      "success": true,
+      "accuracy": 1.0,
+      "fields": "1/1",
+      "hallucinations": 0,
+      "omissions": 0
+    },
+    {
+      "case_id": "patient-003_lab_a1c",
+      "query_type": "lab_trend",
+      "success": true,
+      "accuracy": 1.0,
+      "fields": "4/4",
+      "hallucinations": 0,
+      "omissions": 0
+    }
+  ]
+}

evaluation/reports/eval_report_20260127_174121.txt

@@ -0,0 +1,34 @@
+============================================================
+PRE-VISIT SUMMARY EVALUATION REPORT
+Generated: 2026-01-27 17:41:21
+============================================================
+
+OVERALL RESULTS
+----------------------------------------
+Total Test Cases:     30
+Successful:           30
+Failed:               0
+Success Rate:         100.0%
+
+Total Fields Checked: 128
+Correct Fields:       128
+Field Accuracy:       100.0%
+
+Hallucinations:       0 (0.0%)
+Omissions:            0 (0.0%)
+Mismatches:           0
+
+BY QUERY TYPE
+----------------------------------------
+Query Type                Success      Accuracy     Hall.   
+------------------------------------------------------------
+vital_trend               100.0%       100.0%       0       
+medication_list           100.0%       100.0%       0       
+condition_list            100.0%       100.0%       0       
+allergy_list              100.0%       100.0%       0       
+immunization_list         100.0%       100.0%       0       
+procedure_list            100.0%       100.0%       0       
+encounter_list            100.0%       100.0%       0       
+lab_trend                 100.0%       100.0%       0       
+
+============================================================

evaluation/reports/eval_report_20260127_174147.json

@@ -0,0 +1,380 @@
+{
+  "summary": {
+    "total_cases": 30,
+    "successful_cases": 20,
+    "failed_cases": 10,
+    "success_rate": "66.7%",
+    "field_accuracy": "81.1%",
+    "hallucination_rate": "5.3%",
+    "omission_rate": "5.3%"
+  },
+  "field_level": {
+    "total_fields": 132,
+    "correct_fields": 107,
+    "hallucinations": 7,
+    "omissions": 7,
+    "mismatches": 3
+  },
+  "by_query_type": {
+    "vital_trend": {
+      "total": 6,
+      "successful": 5,
+      "failed": 1,
+      "total_fields": 63,
+      "correct_fields": 56,
+      "hallucinations": 3,
+      "omissions": 1,
+      "success_rate": "83.3%",
+      "field_accuracy": "88.9%"
+    },
+    "medication_list": {
+      "total": 6,
+      "successful": 4,
+      "failed": 2,
+      "total_fields": 19,
+      "correct_fields": 15,
+      "hallucinations": 1,
+      "omissions": 1,
+      "success_rate": "66.7%",
+      "field_accuracy": "78.9%"
+    },
+    "condition_list": {
+      "total": 3,
+      "successful": 1,
+      "failed": 2,
+      "total_fields": 9,
+      "correct_fields": 5,
+      "hallucinations": 0,
+      "omissions": 2,
+      "success_rate": "33.3%",
+      "field_accuracy": "55.6%"
+    },
+    "allergy_list": {
+      "total": 3,
+      "successful": 1,
+      "failed": 2,
+      "total_fields": 8,
+      "correct_fields": 4,
+      "hallucinations": 1,
+      "omissions": 2,
+      "success_rate": "33.3%",
+      "field_accuracy": "50.0%"
+    },
+    "immunization_list": {
+      "total": 3,
+      "successful": 1,
+      "failed": 2,
+      "total_fields": 10,
+      "correct_fields": 6,
+      "hallucinations": 1,
+      "omissions": 1,
+      "success_rate": "33.3%",
+      "field_accuracy": "60.0%"
+    },
+    "procedure_list": {
+      "total": 3,
+      "successful": 2,
+      "failed": 1,
+      "total_fields": 8,
+      "correct_fields": 6,
+      "hallucinations": 1,
+      "omissions": 0,
+      "success_rate": "66.7%",
+      "field_accuracy": "75.0%"
+    },
+    "encounter_list": {
+      "total": 3,
+      "successful": 3,
+      "failed": 0,
+      "total_fields": 3,
+      "correct_fields": 3,
+      "hallucinations": 0,
+      "omissions": 0,
+      "success_rate": "100.0%",
+      "field_accuracy": "100.0%"
+    },
+    "lab_trend": {
+      "total": 3,
+      "successful": 3,
+      "failed": 0,
+      "total_fields": 12,
+      "correct_fields": 12,
+      "hallucinations": 0,
+      "omissions": 0,
+      "success_rate": "100.0%",
+      "field_accuracy": "100.0%"
+    }
+  },
+  "case_results": [
+    {
+      "case_id": "patient-001_vital_blood_pressure",
+      "query_type": "vital_trend",
+      "success": true,
+      "accuracy": 0.9285714285714286,
+      "fields": "13/14",
+      "hallucinations": 1,
+      "omissions": 0
+    },
+    {
+      "case_id": "patient-001_vital_heart_rate",
+      "query_type": "vital_trend",
+      "success": true,
+      "accuracy": 0.8571428571428571,
+      "fields": "6/7",
+      "hallucinations": 0,
+      "omissions": 1
+    },
+    {
+      "case_id": "patient-001_meds_all",
+      "query_type": "medication_list",
+      "success": true,
+      "accuracy": 1.0,
+      "fields": "3/3",
+      "hallucinations": 0,
+      "omissions": 0
+    },
+    {
+      "case_id": "patient-001_meds_active",
+      "query_type": "medication_list",
+      "success": false,
+      "accuracy": 0.5,
+      "fields": "2/4",
+      "hallucinations": 1,
+      "omissions": 0
+    },
+    {
+      "case_id": "patient-001_conditions",
+      "query_type": "condition_list",
+      "success": true,
+      "accuracy": 1.0,
+      "fields": "3/3",
+      "hallucinations": 0,
+      "omissions": 0
+    },
+    {
+      "case_id": "patient-001_allergies",
+      "query_type": "allergy_list",
+      "success": false,
+      "accuracy": 0.3333333333333333,
+      "fields": "1/3",
+      "hallucinations": 1,
+      "omissions": 1
+    },
+    {
+      "case_id": "patient-001_immunizations",
+      "query_type": "immunization_list",
+      "success": false,
+      "accuracy": 0.5,
+      "fields": "2/4",
+      "hallucinations": 1,
+      "omissions": 0
+    },
+    {
+      "case_id": "patient-001_procedures",
+      "query_type": "procedure_list",
+      "success": true,
+      "accuracy": 1.0,
+      "fields": "2/2",
+      "hallucinations": 0,
+      "omissions": 0
+    },
+    {
+      "case_id": "patient-001_encounters",
+      "query_type": "encounter_list",
+      "success": true,
+      "accuracy": 1.0,
+      "fields": "1/1",
+      "hallucinations": 0,
+      "omissions": 0
+    },
+    {
+      "case_id": "patient-001_lab_a1c",
+      "query_type": "lab_trend",
+      "success": true,
+      "accuracy": 1.0,
+      "fields": "4/4",
+      "hallucinations": 0,
+      "omissions": 0
+    },
+    {
+      "case_id": "patient-002_vital_blood_pressure",
+      "query_type": "vital_trend",
+      "success": false,
+      "accuracy": 0.7857142857142857,
+      "fields": "11/14",
+      "hallucinations": 1,
+      "omissions": 0
+    },
+    {
+      "case_id": "patient-002_vital_heart_rate",
+      "query_type": "vital_trend",
+      "success": true,
+      "accuracy": 0.8571428571428571,
+      "fields": "6/7",
+      "hallucinations": 1,
+      "omissions": 0
+    },
+    {
+      "case_id": "patient-002_meds_all",
+      "query_type": "medication_list",
+      "success": true,
+      "accuracy": 1.0,
+      "fields": "3/3",
+      "hallucinations": 0,
+      "omissions": 0
+    },
+    {
+      "case_id": "patient-002_meds_active",
+      "query_type": "medication_list",
+      "success": true,
+      "accuracy": 1.0,
+      "fields": "3/3",
+      "hallucinations": 0,
+      "omissions": 0
+    },
+    {
+      "case_id": "patient-002_conditions",
+      "query_type": "condition_list",
+      "success": false,
+      "accuracy": 0.3333333333333333,
+      "fields": "1/3",
+      "hallucinations": 0,
+      "omissions": 1
+    },
+    {
+      "case_id": "patient-002_allergies",
+      "query_type": "allergy_list",
+      "success": false,
+      "accuracy": 0.3333333333333333,
+      "fields": "1/3",
+      "hallucinations": 0,
+      "omissions": 1
+    },
+    {
+      "case_id": "patient-002_immunizations",
+      "query_type": "immunization_list",
+      "success": false,
+      "accuracy": 0.3333333333333333,
+      "fields": "1/3",
+      "hallucinations": 0,
+      "omissions": 1
+    },
+    {
+      "case_id": "patient-002_procedures",
+      "query_type": "procedure_list",
+      "success": true,
+      "accuracy": 1.0,
+      "fields": "2/2",
+      "hallucinations": 0,
+      "omissions": 0
+    },
+    {
+      "case_id": "patient-002_encounters",
+      "query_type": "encounter_list",
+      "success": true,
+      "accuracy": 1.0,
+      "fields": "1/1",
+      "hallucinations": 0,
+      "omissions": 0
+    },
+    {
+      "case_id": "patient-002_lab_a1c",
+      "query_type": "lab_trend",
+      "success": true,
+      "accuracy": 1.0,
+      "fields": "4/4",
+      "hallucinations": 0,
+      "omissions": 0
+    },
+    {
+      "case_id": "patient-003_vital_blood_pressure",
+      "query_type": "vital_trend",
+      "success": true,
+      "accuracy": 1.0,
+      "fields": "14/14",
+      "hallucinations": 0,
+      "omissions": 0
+    },
+    {
+      "case_id": "patient-003_vital_heart_rate",
+      "query_type": "vital_trend",
+      "success": true,
+      "accuracy": 0.8571428571428571,
+      "fields": "6/7",
+      "hallucinations": 0,
+      "omissions": 0
+    },
+    {
+      "case_id": "patient-003_meds_all",
+      "query_type": "medication_list",
+      "success": false,
+      "accuracy": 0.3333333333333333,
+      "fields": "1/3",
+      "hallucinations": 0,
+      "omissions": 1
+    },
+    {
+      "case_id": "patient-003_meds_active",
+      "query_type": "medication_list",
+      "success": true,
+      "accuracy": 1.0,
+      "fields": "3/3",
+      "hallucinations": 0,
+      "omissions": 0
+    },
+    {
+      "case_id": "patient-003_conditions",
+      "query_type": "condition_list",
+      "success": false,
+      "accuracy": 0.3333333333333333,
+      "fields": "1/3",
+      "hallucinations": 0,
+      "omissions": 1
+    },
+    {
+      "case_id": "patient-003_allergies",
+      "query_type": "allergy_list",
+      "success": true,
+      "accuracy": 1.0,
+      "fields": "2/2",
+      "hallucinations": 0,
+      "omissions": 0
+    },
+    {
+      "case_id": "patient-003_immunizations",
+      "query_type": "immunization_list",
+      "success": true,
+      "accuracy": 1.0,
+      "fields": "3/3",
+      "hallucinations": 0,
+      "omissions": 0
+    },
+    {
+      "case_id": "patient-003_procedures",
+      "query_type": "procedure_list",
+      "success": false,
+      "accuracy": 0.5,
+      "fields": "2/4",
+      "hallucinations": 1,
+      "omissions": 0
+    },
+    {
+      "case_id": "patient-003_encounters",
+      "query_type": "encounter_list",
+      "success": true,
+      "accuracy": 1.0,
+      "fields": "1/1",
+      "hallucinations": 0,
+      "omissions": 0
+    },
+    {
+      "case_id": "patient-003_lab_a1c",
+      "query_type": "lab_trend",
+      "success": true,
+      "accuracy": 1.0,
+      "fields": "4/4",
+      "hallucinations": 0,
+      "omissions": 0
+    }
+  ]
+}

evaluation/reports/eval_report_20260127_174147.txt

@@ -0,0 +1,67 @@
+============================================================
+PRE-VISIT SUMMARY EVALUATION REPORT
+Generated: 2026-01-27 17:41:47
+============================================================
+
+OVERALL RESULTS
+----------------------------------------
+Total Test Cases:     30
+Successful:           20
+Failed:               10
+Success Rate:         66.7%
+
+Total Fields Checked: 132
+Correct Fields:       107
+Field Accuracy:       81.1%
+
+Hallucinations:       7 (5.3%)
+Omissions:            7 (5.3%)
+Mismatches:           3
+
+BY QUERY TYPE
+----------------------------------------
+Query Type                Success      Accuracy     Hall.   
+------------------------------------------------------------
+vital_trend               83.3%        88.9%        3       
+medication_list           66.7%        78.9%        1       
+condition_list            33.3%        55.6%        0       
+allergy_list              33.3%        50.0%        1       
+immunization_list         33.3%        60.0%        1       
+procedure_list            66.7%        75.0%        1       
+encounter_list            100.0%       100.0%       0       
+lab_trend                 100.0%       100.0%       0       
+
+FAILED CASES
+----------------------------------------
+  patient-001_meds_active
+    Type: medication_list, Accuracy: 50.0%
+    Hallucinations: 1, Omissions: 0
+  patient-001_allergies
+    Type: allergy_list, Accuracy: 33.3%
+    Hallucinations: 1, Omissions: 1
+  patient-001_immunizations
+    Type: immunization_list, Accuracy: 50.0%
+    Hallucinations: 1, Omissions: 0
+  patient-002_vital_blood_pressure
+    Type: vital_trend, Accuracy: 78.6%
+    Hallucinations: 1, Omissions: 0
+  patient-002_conditions
+    Type: condition_list, Accuracy: 33.3%
+    Hallucinations: 0, Omissions: 1
+  patient-002_allergies
+    Type: allergy_list, Accuracy: 33.3%
+    Hallucinations: 0, Omissions: 1
+  patient-002_immunizations
+    Type: immunization_list, Accuracy: 33.3%
+    Hallucinations: 0, Omissions: 1
+  patient-003_meds_all
+    Type: medication_list, Accuracy: 33.3%
+    Hallucinations: 0, Omissions: 1
+  patient-003_conditions
+    Type: condition_list, Accuracy: 33.3%
+    Hallucinations: 0, Omissions: 1
+  patient-003_procedures
+    Type: procedure_list, Accuracy: 50.0%
+    Hallucinations: 1, Omissions: 0
+
+============================================================

evaluation/run_evaluation.py

@@ -0,0 +1,246 @@
+#!/usr/bin/env python3
+"""
+Evaluation Runner
+
+Main entry point for running the pre-visit summary evaluation.
+
+This can be run in two modes:
+1. Direct mode: Directly compute expected vs actual from database (no LLM needed)
+2. Agent mode: Run actual agent queries and extract facts from responses
+
+For initial testing, we use direct mode to validate the evaluation framework.
+"""
+
+import os
+import sys
+import json
+import argparse
+from datetime import datetime
+from typing import Dict, List, Any
+
+# Add parent directory to path for imports
+sys.path.insert(0, os.path.dirname(os.path.dirname(os.path.abspath(__file__))))
+
+from evaluation.test_generator import generate_all_test_cases, get_test_summary
+from evaluation.expected_values import compute_expected_values
+from evaluation.evaluator import evaluate_case, CaseEvaluation
+from evaluation.metrics import aggregate_metrics, format_report, save_report
+
+
+def run_direct_evaluation(num_patients: int = 10, output_dir: str = None) -> Dict:
+    """
+    Run evaluation in direct mode.
+    
+    This mode:
+    1. Generates test cases from database
+    2. Computes expected values from database
+    3. Simulates "perfect" agent that returns exactly the expected values
+    4. Computes metrics
+    
+    This validates the evaluation framework works correctly.
+    A perfect agent should score 100%.
+    """
+    print("=" * 60)
+    print("PRE-VISIT SUMMARY EVALUATION - DIRECT MODE")
+    print("=" * 60)
+    print(f"\nGenerating test cases for {num_patients} patients...")
+    
+    # Generate test cases
+    test_cases = generate_all_test_cases(num_patients=num_patients)
+    summary = get_test_summary(test_cases)
+    
+    print(f"Generated {summary['total_cases']} test cases")
+    print("\nBy query type:")
+    for qtype, count in sorted(summary["by_type"].items()):
+        print(f"  {qtype}: {count}")
+    
+    print("\nRunning evaluation...")
+    
+    evaluations = []
+    for i, test_case in enumerate(test_cases):
+        # Compute expected values
+        expected = compute_expected_values(test_case)
+        
+        # In direct mode, actual = expected (simulating perfect agent)
+        actual_facts = expected.copy()
+        
+        # Evaluate
+        evaluation = evaluate_case(test_case, expected, actual_facts)
+        evaluations.append(evaluation)
+        
+        # Progress indicator
+        if (i + 1) % 20 == 0:
+            print(f"  Processed {i + 1}/{len(test_cases)} cases...")
+    
+    # Aggregate metrics
+    metrics = aggregate_metrics(evaluations)
+    
+    # Print report
+    print("\n" + format_report(metrics))
+    
+    # Save report
+    if output_dir:
+        text_path, json_path = save_report(metrics, output_dir)
+        print(f"\nReports saved to:")
+        print(f"  {text_path}")
+        print(f"  {json_path}")
+    
+    return metrics.to_dict()
+
+
+def run_simulated_evaluation(num_patients: int = 10, error_rate: float = 0.1,
+                             output_dir: str = None) -> Dict:
+    """
+    Run evaluation with simulated errors.
+    
+    This mode introduces controlled errors to test that the
+    evaluation framework correctly detects them.
+    
+    Args:
+        num_patients: Number of patients to test
+        error_rate: Fraction of values to corrupt (0.0 - 1.0)
+        output_dir: Directory to save reports
+    """
+    import random
+    
+    print("=" * 60)
+    print("PRE-VISIT SUMMARY EVALUATION - SIMULATED ERROR MODE")
+    print(f"Error rate: {error_rate:.0%}")
+    print("=" * 60)
+    
+    print(f"\nGenerating test cases for {num_patients} patients...")
+    
+    test_cases = generate_all_test_cases(num_patients=num_patients)
+    summary = get_test_summary(test_cases)
+    
+    print(f"Generated {summary['total_cases']} test cases")
+    
+    print("\nRunning evaluation with simulated errors...")
+    
+    evaluations = []
+    for i, test_case in enumerate(test_cases):
+        expected = compute_expected_values(test_case)
+        
+        # Create actual with some errors
+        actual_facts = introduce_errors(expected, error_rate)
+        
+        evaluation = evaluate_case(test_case, expected, actual_facts)
+        evaluations.append(evaluation)
+        
+        if (i + 1) % 20 == 0:
+            print(f"  Processed {i + 1}/{len(test_cases)} cases...")
+    
+    metrics = aggregate_metrics(evaluations)
+    
+    print("\n" + format_report(metrics))
+    
+    if output_dir:
+        text_path, json_path = save_report(metrics, output_dir)
+        print(f"\nReports saved to:")
+        print(f"  {text_path}")
+        print(f"  {json_path}")
+    
+    return metrics.to_dict()
+
+
+def introduce_errors(expected: Dict, error_rate: float) -> Dict:
+    """
+    Introduce controlled errors into expected values.
+    
+    Error types:
+    - Numeric perturbation (add/subtract random amount)
+    - Omission (remove items from lists)
+    - Hallucination (add fake items to lists)
+    """
+    import random
+    import copy
+    
+    actual = copy.deepcopy(expected)
+    
+    # Handle metrics dict (for vital/lab trends)
+    if "metrics" in actual:
+        for label, label_metrics in actual["metrics"].items():
+            if isinstance(label_metrics, dict):
+                for key, value in list(label_metrics.items()):
+                    if random.random() < error_rate:
+                        if isinstance(value, (int, float)) and key != "count":
+                            # Numeric perturbation
+                            label_metrics[key] = round(value + random.uniform(-5, 5), 1)
+                        elif key == "count" and random.random() < 0.5:
+                            # Sometimes omit count
+                            label_metrics[key] = None
+    
+    # Handle list fields
+    for list_key in ["medication_names", "condition_names", "substances", 
+                      "vaccine_names", "procedure_names"]:
+        if list_key in actual:
+            items = actual[list_key]
+            new_items = []
+            
+            for item in items:
+                if random.random() < error_rate:
+                    # Omit this item
+                    continue
+                new_items.append(item)
+            
+            # Maybe add hallucination
+            if random.random() < error_rate:
+                new_items.append(f"FAKE_ITEM_{random.randint(1000, 9999)}")
+            
+            actual[list_key] = new_items
+            actual["count"] = len(new_items)
+    
+    return actual
+
+
+def main():
+    parser = argparse.ArgumentParser(description="Run pre-visit summary evaluation")
+    
+    parser.add_argument(
+        "--mode", 
+        choices=["direct", "simulated"],
+        default="direct",
+        help="Evaluation mode: 'direct' for perfect agent, 'simulated' for errors"
+    )
+    
+    parser.add_argument(
+        "--patients", 
+        type=int, 
+        default=10,
+        help="Number of patients to test (default: 10)"
+    )
+    
+    parser.add_argument(
+        "--error-rate",
+        type=float,
+        default=0.1,
+        help="Error rate for simulated mode (default: 0.1)"
+    )
+    
+    parser.add_argument(
+        "--output-dir",
+        type=str,
+        default="evaluation/reports",
+        help="Directory to save reports (default: evaluation/reports)"
+    )
+    
+    args = parser.parse_args()
+    
+    # Ensure output directory exists
+    os.makedirs(args.output_dir, exist_ok=True)
+    
+    if args.mode == "direct":
+        run_direct_evaluation(
+            num_patients=args.patients,
+            output_dir=args.output_dir
+        )
+    else:
+        run_simulated_evaluation(
+            num_patients=args.patients,
+            error_rate=args.error_rate,
+            output_dir=args.output_dir
+        )
+
+
+if __name__ == "__main__":
+    main()

evaluation/test_generator.py

@@ -0,0 +1,357 @@
+#!/usr/bin/env python3
+"""
+Test Case Generator for Pre-Visit Summary Evaluation
+
+Generates test cases from Synthea patient data with known ground truth.
+"""
+
+import sqlite3
+import random
+from datetime import datetime, timedelta
+from typing import List, Dict, Any
+import os
+
+DB_PATH = os.getenv("DB_PATH", "data/fhir.db")
+
+
+def get_db():
+    """Get database connection."""
+    conn = sqlite3.connect(DB_PATH)
+    conn.row_factory = sqlite3.Row
+    return conn
+
+
+def get_test_patients(limit: int = 10) -> List[Dict]:
+    """Get patients that have sufficient data for testing."""
+    conn = get_db()
+    try:
+        # Find patients with good data coverage
+        cursor = conn.execute("""
+            SELECT p.id, p.given_name, p.family_name, p.birth_date, p.gender,
+                   (SELECT COUNT(*) FROM conditions WHERE patient_id = p.id) as condition_count,
+                   (SELECT COUNT(*) FROM medications WHERE patient_id = p.id) as med_count,
+                   (SELECT COUNT(*) FROM observations WHERE patient_id = p.id) as obs_count,
+                   (SELECT COUNT(*) FROM allergies WHERE patient_id = p.id) as allergy_count,
+                   (SELECT COUNT(*) FROM immunizations WHERE patient_id = p.id) as imm_count,
+                   (SELECT COUNT(*) FROM procedures WHERE patient_id = p.id) as proc_count,
+                   (SELECT COUNT(*) FROM encounters WHERE patient_id = p.id) as enc_count
+            FROM patients p
+            WHERE (SELECT COUNT(*) FROM observations WHERE patient_id = p.id) > 10
+            ORDER BY obs_count DESC
+            LIMIT ?
+        """, (limit,))
+        
+        patients = []
+        for row in cursor.fetchall():
+            patients.append({
+                "patient_id": row["id"],
+                "name": f"{row['given_name']} {row['family_name']}",
+                "birth_date": row["birth_date"],
+                "gender": row["gender"],
+                "data_counts": {
+                    "conditions": row["condition_count"],
+                    "medications": row["med_count"],
+                    "observations": row["obs_count"],
+                    "allergies": row["allergy_count"],
+                    "immunizations": row["imm_count"],
+                    "procedures": row["proc_count"],
+                    "encounters": row["enc_count"]
+                }
+            })
+        return patients
+    finally:
+        conn.close()
+
+
+def generate_vital_trend_cases(patient_id: str, days: int = 30) -> List[Dict]:
+    """Generate test cases for vital sign trends (BP, heart rate, etc.)."""
+    test_cases = []
+    
+    vital_types = [
+        ("blood_pressure", ["8480-6", "8462-4"], ["systolic", "diastolic"]),
+        ("heart_rate", ["8867-4"], ["heart_rate"]),
+        ("weight", ["29463-7"], ["weight"]),
+        ("temperature", ["8310-5"], ["temperature"]),
+        ("oxygen_saturation", ["2708-6"], ["oxygen_saturation"]),
+    ]
+    
+    conn = get_db()
+    try:
+        for vital_name, codes, labels in vital_types:
+            # Check if patient has this vital data
+            placeholders = ",".join(["?" for _ in codes])
+            cursor = conn.execute(f"""
+                SELECT COUNT(*) as cnt FROM observations
+                WHERE patient_id = ? AND code IN ({placeholders})
+            """, [patient_id] + codes)
+            
+            count = cursor.fetchone()["cnt"]
+            if count >= 3:  # Need at least 3 readings for meaningful test
+                test_cases.append({
+                    "case_id": f"{patient_id}_vital_{vital_name}",
+                    "patient_id": patient_id,
+                    "query_type": "vital_trend",
+                    "query": f"Show me my {vital_name.replace('_', ' ')} chart",
+                    "parameters": {
+                        "vital_type": vital_name,
+                        "days": days,
+                        "codes": codes,
+                        "labels": labels
+                    }
+                })
+    finally:
+        conn.close()
+    
+    return test_cases
+
+
+def generate_medication_cases(patient_id: str) -> List[Dict]:
+    """Generate test cases for medication queries."""
+    test_cases = []
+    
+    conn = get_db()
+    try:
+        # Check if patient has medications
+        cursor = conn.execute("""
+            SELECT COUNT(*) as total,
+                   SUM(CASE WHEN status = 'active' THEN 1 ELSE 0 END) as active
+            FROM medications WHERE patient_id = ?
+        """, (patient_id,))
+        
+        row = cursor.fetchone()
+        if row["total"] > 0:
+            # All medications
+            test_cases.append({
+                "case_id": f"{patient_id}_meds_all",
+                "patient_id": patient_id,
+                "query_type": "medication_list",
+                "query": "What medications am I taking?",
+                "parameters": {"status": None}
+            })
+            
+            # Active only
+            if row["active"] > 0:
+                test_cases.append({
+                    "case_id": f"{patient_id}_meds_active",
+                    "patient_id": patient_id,
+                    "query_type": "medication_list",
+                    "query": "What are my current active medications?",
+                    "parameters": {"status": "active"}
+                })
+    finally:
+        conn.close()
+    
+    return test_cases
+
+
+def generate_condition_cases(patient_id: str) -> List[Dict]:
+    """Generate test cases for condition queries."""
+    test_cases = []
+    
+    conn = get_db()
+    try:
+        cursor = conn.execute("""
+            SELECT COUNT(*) as cnt FROM conditions WHERE patient_id = ?
+        """, (patient_id,))
+        
+        if cursor.fetchone()["cnt"] > 0:
+            test_cases.append({
+                "case_id": f"{patient_id}_conditions",
+                "patient_id": patient_id,
+                "query_type": "condition_list",
+                "query": "What are my medical conditions?",
+                "parameters": {}
+            })
+    finally:
+        conn.close()
+    
+    return test_cases
+
+
+def generate_allergy_cases(patient_id: str) -> List[Dict]:
+    """Generate test cases for allergy queries."""
+    test_cases = []
+    
+    conn = get_db()
+    try:
+        cursor = conn.execute("""
+            SELECT COUNT(*) as cnt FROM allergies WHERE patient_id = ?
+        """, (patient_id,))
+        
+        if cursor.fetchone()["cnt"] > 0:
+            test_cases.append({
+                "case_id": f"{patient_id}_allergies",
+                "patient_id": patient_id,
+                "query_type": "allergy_list",
+                "query": "What are my allergies?",
+                "parameters": {}
+            })
+    finally:
+        conn.close()
+    
+    return test_cases
+
+
+def generate_immunization_cases(patient_id: str) -> List[Dict]:
+    """Generate test cases for immunization queries."""
+    test_cases = []
+    
+    conn = get_db()
+    try:
+        cursor = conn.execute("""
+            SELECT COUNT(*) as cnt FROM immunizations WHERE patient_id = ?
+        """, (patient_id,))
+        
+        if cursor.fetchone()["cnt"] > 0:
+            test_cases.append({
+                "case_id": f"{patient_id}_immunizations",
+                "patient_id": patient_id,
+                "query_type": "immunization_list",
+                "query": "What immunizations have I had?",
+                "parameters": {}
+            })
+    finally:
+        conn.close()
+    
+    return test_cases
+
+
+def generate_procedure_cases(patient_id: str) -> List[Dict]:
+    """Generate test cases for procedure/surgical history queries."""
+    test_cases = []
+    
+    conn = get_db()
+    try:
+        cursor = conn.execute("""
+            SELECT COUNT(*) as cnt FROM procedures WHERE patient_id = ?
+        """, (patient_id,))
+        
+        if cursor.fetchone()["cnt"] > 0:
+            test_cases.append({
+                "case_id": f"{patient_id}_procedures",
+                "patient_id": patient_id,
+                "query_type": "procedure_list",
+                "query": "What procedures or surgeries have I had?",
+                "parameters": {}
+            })
+    finally:
+        conn.close()
+    
+    return test_cases
+
+
+def generate_encounter_cases(patient_id: str) -> List[Dict]:
+    """Generate test cases for encounter history queries."""
+    test_cases = []
+    
+    conn = get_db()
+    try:
+        cursor = conn.execute("""
+            SELECT COUNT(*) as cnt FROM encounters WHERE patient_id = ?
+        """, (patient_id,))
+        
+        if cursor.fetchone()["cnt"] > 0:
+            test_cases.append({
+                "case_id": f"{patient_id}_encounters",
+                "patient_id": patient_id,
+                "query_type": "encounter_list",
+                "query": "Show me my recent visits",
+                "parameters": {"limit": 5}
+            })
+    finally:
+        conn.close()
+    
+    return test_cases
+
+
+def generate_lab_cases(patient_id: str) -> List[Dict]:
+    """Generate test cases for lab result queries."""
+    test_cases = []
+    
+    lab_types = [
+        ("a1c", "4548-4", "HbA1c"),
+        ("cholesterol", "2093-3", "Total Cholesterol"),
+        ("glucose", "2345-7", "Glucose"),
+    ]
+    
+    conn = get_db()
+    try:
+        for lab_name, code, display in lab_types:
+            cursor = conn.execute("""
+                SELECT COUNT(*) as cnt FROM observations
+                WHERE patient_id = ? AND code = ?
+            """, (patient_id, code))
+            
+            if cursor.fetchone()["cnt"] >= 2:
+                test_cases.append({
+                    "case_id": f"{patient_id}_lab_{lab_name}",
+                    "patient_id": patient_id,
+                    "query_type": "lab_trend",
+                    "query": f"Show me my {display} history",
+                    "parameters": {
+                        "lab_type": lab_name,
+                        "code": code
+                    }
+                })
+    finally:
+        conn.close()
+    
+    return test_cases
+
+
+def generate_all_test_cases(num_patients: int = 10) -> List[Dict]:
+    """Generate complete test suite from available patients."""
+    patients = get_test_patients(num_patients)
+    all_cases = []
+    
+    for patient in patients:
+        pid = patient["patient_id"]
+        
+        # Generate cases for each data type
+        all_cases.extend(generate_vital_trend_cases(pid))
+        all_cases.extend(generate_medication_cases(pid))
+        all_cases.extend(generate_condition_cases(pid))
+        all_cases.extend(generate_allergy_cases(pid))
+        all_cases.extend(generate_immunization_cases(pid))
+        all_cases.extend(generate_procedure_cases(pid))
+        all_cases.extend(generate_encounter_cases(pid))
+        all_cases.extend(generate_lab_cases(pid))
+    
+    return all_cases
+
+
+def get_test_summary(test_cases: List[Dict]) -> Dict:
+    """Get summary of generated test cases."""
+    summary = {
+        "total_cases": len(test_cases),
+        "by_type": {},
+        "by_patient": {}
+    }
+    
+    for case in test_cases:
+        # Count by type
+        qtype = case["query_type"]
+        summary["by_type"][qtype] = summary["by_type"].get(qtype, 0) + 1
+        
+        # Count by patient
+        pid = case["patient_id"]
+        summary["by_patient"][pid] = summary["by_patient"].get(pid, 0) + 1
+    
+    return summary
+
+
+if __name__ == "__main__":
+    # Test the generator
+    print("Generating test cases...")
+    cases = generate_all_test_cases(num_patients=5)
+    summary = get_test_summary(cases)
+    
+    print(f"\nTotal test cases: {summary['total_cases']}")
+    print("\nBy query type:")
+    for qtype, count in sorted(summary["by_type"].items()):
+        print(f"  {qtype}: {count}")
+    
+    print("\nSample test case:")
+    if cases:
+        import json
+        print(json.dumps(cases[0], indent=2))