run_eval.py

"""
Evaluation script for DeepPragma Classifier.

Runs the labeled test set against GroqClient.classify() and reports:
- overall accuracy
- per-category accuracy
- a confusion list of misclassified examples (for manual inspection)

Usage:
    python run_eval.py

Requires GROQ_API_KEY to be set in the environment.
Place this file at the project root (same level as app.py),
and test_set.json inside an eval/ folder (or adjust TEST_SET_PATH below).
"""

import json
import os
import time
from collections import defaultdict

from src.llm_client import GroqClient

TEST_SET_PATH = os.path.join("eval", "test_set.json")


def load_test_set(path):
    with open(path, "r", encoding="utf-8") as f:
        return json.load(f)


def run_evaluation():
    if not os.environ.get("GROQ_API_KEY"):
        print("ERROR: GROQ_API_KEY not set in environment.")
        return

    test_cases = load_test_set(TEST_SET_PATH)
    llm = GroqClient()

    total = len(test_cases)
    correct = 0
    per_category_total = defaultdict(int)
    per_category_correct = defaultdict(int)
    errors = []

    for i, case in enumerate(test_cases, start=1):
        text = case["text"]
        expected = case["expected"]

        result = llm.classify(text)
        predicted = result["label"]

        per_category_total[expected] += 1
        is_correct = predicted == expected
        if is_correct:
            correct += 1
            per_category_correct[expected] += 1
        else:
            errors.append({
                "text": text,
                "expected": expected,
                "predicted": predicted,
                "reasoning": result.get("reasoning", ""),
            })

        print(f"[{i}/{total}] {'OK ' if is_correct else 'FAIL'} "
              f"expected={expected!r} predicted={predicted!r}")

        # small delay to be gentle with API rate limits
        time.sleep(0.5)

    print("\n" + "=" * 60)
    print(f"OVERALL ACCURACY: {correct}/{total} ({100 * correct / total:.1f}%)")
    print("=" * 60)

    print("\nPER-CATEGORY ACCURACY:")
    for category in per_category_total:
        cat_total = per_category_total[category]
        cat_correct = per_category_correct[category]
        print(f"  {category}: {cat_correct}/{cat_total} "
              f"({100 * cat_correct / cat_total:.1f}%)")

    if errors:
        print(f"\nMISCLASSIFIED EXAMPLES ({len(errors)}):")
        for err in errors:
            print(f"\n  Text: {err['text']}")
            print(f"  Expected:  {err['expected']}")
            print(f"  Predicted: {err['predicted']}")
            print(f"  Reasoning: {err['reasoning']}")

    # Save results to a JSON report for later reference (e.g. in your CV/interview prep)
    report = {
        "total": total,
        "correct": correct,
        "accuracy": round(100 * correct / total, 1),
        "per_category": {
            cat: {
                "total": per_category_total[cat],
                "correct": per_category_correct[cat],
                "accuracy": round(100 * per_category_correct[cat] / per_category_total[cat], 1),
            }
            for cat in per_category_total
        },
        "errors": errors,
    }
    with open("eval_report.json", "w", encoding="utf-8") as f:
        json.dump(report, f, indent=2, ensure_ascii=False)
    print("\nFull report saved to eval_report.json")


if __name__ == "__main__":
    run_evaluation()