Spaces:
Sleeping
Sleeping
File size: 3,513 Bytes
2d7f235 | 1 2 3 4 5 6 7 8 9 10 11 12 13 14 15 16 17 18 19 20 21 22 23 24 25 26 27 28 29 30 31 32 33 34 35 36 37 38 39 40 41 42 43 44 45 46 47 48 49 50 51 52 53 54 55 56 57 58 59 60 61 62 63 64 65 66 67 68 69 70 71 72 73 74 75 76 77 78 79 80 81 82 83 84 85 86 87 88 89 90 91 92 93 94 95 96 97 98 99 100 101 102 103 104 105 106 107 108 109 110 111 112 | """
Evaluation script for DeepPragma Classifier.
Runs the labeled test set against GroqClient.classify() and reports:
- overall accuracy
- per-category accuracy
- a confusion list of misclassified examples (for manual inspection)
Usage:
python run_eval.py
Requires GROQ_API_KEY to be set in the environment.
Place this file at the project root (same level as app.py),
and test_set.json inside an eval/ folder (or adjust TEST_SET_PATH below).
"""
import json
import os
import time
from collections import defaultdict
from src.llm_client import GroqClient
TEST_SET_PATH = os.path.join("eval", "test_set.json")
def load_test_set(path):
with open(path, "r", encoding="utf-8") as f:
return json.load(f)
def run_evaluation():
if not os.environ.get("GROQ_API_KEY"):
print("ERROR: GROQ_API_KEY not set in environment.")
return
test_cases = load_test_set(TEST_SET_PATH)
llm = GroqClient()
total = len(test_cases)
correct = 0
per_category_total = defaultdict(int)
per_category_correct = defaultdict(int)
errors = []
for i, case in enumerate(test_cases, start=1):
text = case["text"]
expected = case["expected"]
result = llm.classify(text)
predicted = result["label"]
per_category_total[expected] += 1
is_correct = predicted == expected
if is_correct:
correct += 1
per_category_correct[expected] += 1
else:
errors.append({
"text": text,
"expected": expected,
"predicted": predicted,
"reasoning": result.get("reasoning", ""),
})
print(f"[{i}/{total}] {'OK ' if is_correct else 'FAIL'} "
f"expected={expected!r} predicted={predicted!r}")
# small delay to be gentle with API rate limits
time.sleep(0.5)
print("\n" + "=" * 60)
print(f"OVERALL ACCURACY: {correct}/{total} ({100 * correct / total:.1f}%)")
print("=" * 60)
print("\nPER-CATEGORY ACCURACY:")
for category in per_category_total:
cat_total = per_category_total[category]
cat_correct = per_category_correct[category]
print(f" {category}: {cat_correct}/{cat_total} "
f"({100 * cat_correct / cat_total:.1f}%)")
if errors:
print(f"\nMISCLASSIFIED EXAMPLES ({len(errors)}):")
for err in errors:
print(f"\n Text: {err['text']}")
print(f" Expected: {err['expected']}")
print(f" Predicted: {err['predicted']}")
print(f" Reasoning: {err['reasoning']}")
# Save results to a JSON report for later reference (e.g. in your CV/interview prep)
report = {
"total": total,
"correct": correct,
"accuracy": round(100 * correct / total, 1),
"per_category": {
cat: {
"total": per_category_total[cat],
"correct": per_category_correct[cat],
"accuracy": round(100 * per_category_correct[cat] / per_category_total[cat], 1),
}
for cat in per_category_total
},
"errors": errors,
}
with open("eval_report.json", "w", encoding="utf-8") as f:
json.dump(report, f, indent=2, ensure_ascii=False)
print("\nFull report saved to eval_report.json")
if __name__ == "__main__":
run_evaluation() |