Spaces:
Sleeping
Sleeping
Commit ·
201ef1b
1
Parent(s): 3a95f02
Fix: clamp grader scores to strictly (0, 1) — evaluator requires 0 < score < 1
Browse files
invoice_agent/graders/__init__.py
CHANGED
|
@@ -26,6 +26,11 @@ def _field_match(extracted: str, truth: str) -> float:
|
|
| 26 |
return 0.0
|
| 27 |
|
| 28 |
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
| 29 |
def grade_easy(
|
| 30 |
extracted_fields: Dict[str, str],
|
| 31 |
flagged_discrepancies: List[Dict[str, str]],
|
|
@@ -33,11 +38,11 @@ def grade_easy(
|
|
| 33 |
ground_truth_discrepancies: List[Dict[str, str]],
|
| 34 |
) -> float:
|
| 35 |
"""Grade Task 1 (Easy): Clean invoice extraction.
|
| 36 |
-
|
| 37 |
Score = average field accuracy across all required fields.
|
| 38 |
"""
|
| 39 |
if not ground_truth_fields:
|
| 40 |
-
return 0.0
|
| 41 |
|
| 42 |
total = 0.0
|
| 43 |
count = len(ground_truth_fields)
|
|
@@ -47,7 +52,7 @@ def grade_easy(
|
|
| 47 |
total += _field_match(extracted_value, truth_value)
|
| 48 |
|
| 49 |
score = total / count if count > 0 else 0.0
|
| 50 |
-
return
|
| 51 |
|
| 52 |
|
| 53 |
def grade_medium(
|
|
@@ -63,7 +68,7 @@ def grade_medium(
|
|
| 63 |
+ calibration_bonus (up to 0.05)
|
| 64 |
"""
|
| 65 |
if not ground_truth_fields:
|
| 66 |
-
return 0.0
|
| 67 |
|
| 68 |
# --- Field accuracy (50%) ---
|
| 69 |
field_total = 0.0
|
|
@@ -87,7 +92,7 @@ def grade_medium(
|
|
| 87 |
cal_bonus = 0.05 * cal_score
|
| 88 |
|
| 89 |
score = 0.50 * field_accuracy + 0.40 * disc_f1 + 0.10 * efficiency + cal_bonus
|
| 90 |
-
return
|
| 91 |
|
| 92 |
|
| 93 |
def grade_hard(
|
|
@@ -107,7 +112,7 @@ def grade_hard(
|
|
| 107 |
(quantity shortfalls, damaged goods, unreceived items).
|
| 108 |
"""
|
| 109 |
if not ground_truth_fields:
|
| 110 |
-
return 0.0
|
| 111 |
|
| 112 |
# --- Field accuracy (25%) ---
|
| 113 |
field_total = 0.0
|
|
@@ -160,7 +165,7 @@ def grade_hard(
|
|
| 160 |
- fp_penalty
|
| 161 |
+ cal_bonus
|
| 162 |
)
|
| 163 |
-
return
|
| 164 |
|
| 165 |
|
| 166 |
def compute_calibration(confidence_records: List[Dict]) -> float:
|
|
|
|
| 26 |
return 0.0
|
| 27 |
|
| 28 |
|
| 29 |
+
def _clamp(score: float) -> float:
|
| 30 |
+
"""Clamp score to strictly (0, 1) as required by the evaluator."""
|
| 31 |
+
return round(min(max(score, 0.001), 0.999), 4)
|
| 32 |
+
|
| 33 |
+
|
| 34 |
def grade_easy(
|
| 35 |
extracted_fields: Dict[str, str],
|
| 36 |
flagged_discrepancies: List[Dict[str, str]],
|
|
|
|
| 38 |
ground_truth_discrepancies: List[Dict[str, str]],
|
| 39 |
) -> float:
|
| 40 |
"""Grade Task 1 (Easy): Clean invoice extraction.
|
| 41 |
+
|
| 42 |
Score = average field accuracy across all required fields.
|
| 43 |
"""
|
| 44 |
if not ground_truth_fields:
|
| 45 |
+
return _clamp(0.0)
|
| 46 |
|
| 47 |
total = 0.0
|
| 48 |
count = len(ground_truth_fields)
|
|
|
|
| 52 |
total += _field_match(extracted_value, truth_value)
|
| 53 |
|
| 54 |
score = total / count if count > 0 else 0.0
|
| 55 |
+
return _clamp(score)
|
| 56 |
|
| 57 |
|
| 58 |
def grade_medium(
|
|
|
|
| 68 |
+ calibration_bonus (up to 0.05)
|
| 69 |
"""
|
| 70 |
if not ground_truth_fields:
|
| 71 |
+
return _clamp(0.0)
|
| 72 |
|
| 73 |
# --- Field accuracy (50%) ---
|
| 74 |
field_total = 0.0
|
|
|
|
| 92 |
cal_bonus = 0.05 * cal_score
|
| 93 |
|
| 94 |
score = 0.50 * field_accuracy + 0.40 * disc_f1 + 0.10 * efficiency + cal_bonus
|
| 95 |
+
return _clamp(score)
|
| 96 |
|
| 97 |
|
| 98 |
def grade_hard(
|
|
|
|
| 112 |
(quantity shortfalls, damaged goods, unreceived items).
|
| 113 |
"""
|
| 114 |
if not ground_truth_fields:
|
| 115 |
+
return _clamp(0.0)
|
| 116 |
|
| 117 |
# --- Field accuracy (25%) ---
|
| 118 |
field_total = 0.0
|
|
|
|
| 165 |
- fp_penalty
|
| 166 |
+ cal_bonus
|
| 167 |
)
|
| 168 |
+
return _clamp(score)
|
| 169 |
|
| 170 |
|
| 171 |
def compute_calibration(confidence_records: List[Dict]) -> float:
|