akashkathole Claude Sonnet 4.6 commited on
Commit
201ef1b
·
1 Parent(s): 3a95f02

Fix: clamp grader scores to strictly (0, 1) — evaluator requires 0 < score < 1

Browse files
Files changed (1) hide show
  1. invoice_agent/graders/__init__.py +12 -7
invoice_agent/graders/__init__.py CHANGED
@@ -26,6 +26,11 @@ def _field_match(extracted: str, truth: str) -> float:
26
  return 0.0
27
 
28
 
 
 
 
 
 
29
  def grade_easy(
30
  extracted_fields: Dict[str, str],
31
  flagged_discrepancies: List[Dict[str, str]],
@@ -33,11 +38,11 @@ def grade_easy(
33
  ground_truth_discrepancies: List[Dict[str, str]],
34
  ) -> float:
35
  """Grade Task 1 (Easy): Clean invoice extraction.
36
-
37
  Score = average field accuracy across all required fields.
38
  """
39
  if not ground_truth_fields:
40
- return 0.0
41
 
42
  total = 0.0
43
  count = len(ground_truth_fields)
@@ -47,7 +52,7 @@ def grade_easy(
47
  total += _field_match(extracted_value, truth_value)
48
 
49
  score = total / count if count > 0 else 0.0
50
- return round(min(max(score, 0.0), 1.0), 4)
51
 
52
 
53
  def grade_medium(
@@ -63,7 +68,7 @@ def grade_medium(
63
  + calibration_bonus (up to 0.05)
64
  """
65
  if not ground_truth_fields:
66
- return 0.0
67
 
68
  # --- Field accuracy (50%) ---
69
  field_total = 0.0
@@ -87,7 +92,7 @@ def grade_medium(
87
  cal_bonus = 0.05 * cal_score
88
 
89
  score = 0.50 * field_accuracy + 0.40 * disc_f1 + 0.10 * efficiency + cal_bonus
90
- return round(min(max(score, 0.0), 1.0), 4)
91
 
92
 
93
  def grade_hard(
@@ -107,7 +112,7 @@ def grade_hard(
107
  (quantity shortfalls, damaged goods, unreceived items).
108
  """
109
  if not ground_truth_fields:
110
- return 0.0
111
 
112
  # --- Field accuracy (25%) ---
113
  field_total = 0.0
@@ -160,7 +165,7 @@ def grade_hard(
160
  - fp_penalty
161
  + cal_bonus
162
  )
163
- return round(min(max(score, 0.0), 1.0), 4)
164
 
165
 
166
  def compute_calibration(confidence_records: List[Dict]) -> float:
 
26
  return 0.0
27
 
28
 
29
+ def _clamp(score: float) -> float:
30
+ """Clamp score to strictly (0, 1) as required by the evaluator."""
31
+ return round(min(max(score, 0.001), 0.999), 4)
32
+
33
+
34
  def grade_easy(
35
  extracted_fields: Dict[str, str],
36
  flagged_discrepancies: List[Dict[str, str]],
 
38
  ground_truth_discrepancies: List[Dict[str, str]],
39
  ) -> float:
40
  """Grade Task 1 (Easy): Clean invoice extraction.
41
+
42
  Score = average field accuracy across all required fields.
43
  """
44
  if not ground_truth_fields:
45
+ return _clamp(0.0)
46
 
47
  total = 0.0
48
  count = len(ground_truth_fields)
 
52
  total += _field_match(extracted_value, truth_value)
53
 
54
  score = total / count if count > 0 else 0.0
55
+ return _clamp(score)
56
 
57
 
58
  def grade_medium(
 
68
  + calibration_bonus (up to 0.05)
69
  """
70
  if not ground_truth_fields:
71
+ return _clamp(0.0)
72
 
73
  # --- Field accuracy (50%) ---
74
  field_total = 0.0
 
92
  cal_bonus = 0.05 * cal_score
93
 
94
  score = 0.50 * field_accuracy + 0.40 * disc_f1 + 0.10 * efficiency + cal_bonus
95
+ return _clamp(score)
96
 
97
 
98
  def grade_hard(
 
112
  (quantity shortfalls, damaged goods, unreceived items).
113
  """
114
  if not ground_truth_fields:
115
+ return _clamp(0.0)
116
 
117
  # --- Field accuracy (25%) ---
118
  field_total = 0.0
 
165
  - fp_penalty
166
  + cal_bonus
167
  )
168
+ return _clamp(score)
169
 
170
 
171
  def compute_calibration(confidence_records: List[Dict]) -> float: