balloonmann commited on
Commit
f76bbd6
·
1 Parent(s): 4943c0a

Refine comment wording for scoring stability paths

Browse files
financial_audit_env/server/app.py CHANGED
@@ -305,13 +305,13 @@ async def get_grader_score(session_id: Optional[str] = None):
305
  }
306
 
307
  def final_clamp(val: float) -> float:
308
- """Guarantee score-like fields are always strictly within (0, 1)."""
309
  return max(0.01, min(0.99, val))
310
 
311
  return {
312
  "status": "completed",
313
  "task_id": env.state.task_id,
314
- # Primary score fields with final clamping for validator safety
315
  "score": final_clamp(result["score"]),
316
  "precision": final_clamp(result["precision"]),
317
  "recall": final_clamp(result["recall"]),
@@ -319,7 +319,7 @@ async def get_grader_score(session_id: Optional[str] = None):
319
  "false_positives": result["false_positives"],
320
  "false_negatives": result["false_negatives"],
321
  "total_errors": result["total_errors"],
322
- # Enhanced scoring fields with final clamping for validator safety
323
  "weighted_score": final_clamp(result.get("weighted_score", result["score"])),
324
  "partial_credit_score": final_clamp(result.get("partial_credit_score", result["score"])),
325
  "partial_matches": result.get("partial_matches", 0),
 
305
  }
306
 
307
  def final_clamp(val: float) -> float:
308
+ """Keep score-like fields within a stable open interval."""
309
  return max(0.01, min(0.99, val))
310
 
311
  return {
312
  "status": "completed",
313
  "task_id": env.state.task_id,
314
+ # Primary score fields with a final stability clamp.
315
  "score": final_clamp(result["score"]),
316
  "precision": final_clamp(result["precision"]),
317
  "recall": final_clamp(result["recall"]),
 
319
  "false_positives": result["false_positives"],
320
  "false_negatives": result["false_negatives"],
321
  "total_errors": result["total_errors"],
322
+ # Enhanced scoring fields with the same stability clamp.
323
  "weighted_score": final_clamp(result.get("weighted_score", result["score"])),
324
  "partial_credit_score": final_clamp(result.get("partial_credit_score", result["score"])),
325
  "partial_matches": result.get("partial_matches", 0),
financial_audit_env/server/environment.py CHANGED
@@ -234,14 +234,13 @@ class FinancialAuditEnvironment(Environment):
234
  # Check if episode should end
235
  is_final = action.submit_final or step_num >= self._task["max_steps"]
236
 
237
- # Compute step reward
238
- # We MUST ensure the sum of all step_rewards across the episode is exactly the final score.
239
- # Otherwise, the evaluator's `sum(rewards)` will exceed 1.0.
240
  if not is_final:
241
  step_reward = 0.0
242
  else:
243
  final_grader = compute_f1_score(self._findings, self._ground_truth)
244
- # The exact clamped F1 score
245
  step_reward = max(0.01, min(0.99, final_grader["score"]))
246
 
247
  self._episode_reward += step_reward
 
234
  # Check if episode should end
235
  is_final = action.submit_final or step_num >= self._task["max_steps"]
236
 
237
+ # Compute step reward.
238
+ # Keep cumulative episode rewards aligned with the final task score.
 
239
  if not is_final:
240
  step_reward = 0.0
241
  else:
242
  final_grader = compute_f1_score(self._findings, self._ground_truth)
243
+ # Use the bounded final score as the terminal reward.
244
  step_reward = max(0.01, min(0.99, final_grader["score"]))
245
 
246
  self._episode_reward += step_reward
financial_audit_env/server/graders.py CHANGED
@@ -20,13 +20,13 @@ from .data_generator import ERROR_MONETARY_VALUES, ERROR_SEVERITY_WEIGHTS
20
 
21
 
22
  # ---------------------------------------------------------------------------
23
- # Phase-2 validator requires every task score to be strictly in (0, 1).
24
- # We enforce: final_score = clamp(round(raw_score, N))
25
  # ---------------------------------------------------------------------------
26
  _SCORE_EPSILON = 0.01
27
 
28
  def _clamp_score(score: float) -> float:
29
- """Clamp a score to be strictly within [0.01, 0.99]."""
30
  if score <= 0.01:
31
  return 0.01
32
  elif score >= 0.99:
@@ -34,7 +34,7 @@ def _clamp_score(score: float) -> float:
34
  return score
35
 
36
  def strict_round_clamp(raw_score: float, n_digits: int = 2) -> float:
37
- """Safely round then clamp to guarantee the result is strictly in [0.01, 0.99]."""
38
  rounded = round(raw_score, n_digits)
39
  if rounded <= 0.01:
40
  return 0.01
@@ -56,7 +56,7 @@ def compute_f1_score(
56
  - score: float (0, 1) exclusive (unweighted F1 — primary metric)
57
  - weighted_score: float (0, 1) exclusive (severity-weighted F1)
58
  - partial_credit_score: float (0, 1) exclusive (with partial credit)
59
- - precision/recall: standard metrics, clamped to (0, 1) exclusive
60
  - true_positives/false_positives/false_negatives: counts
61
  - total_findings/total_errors: counts
62
  - matched_errors/missed_errors/false_positive_list: details
@@ -234,7 +234,7 @@ def compute_f1_score(
234
  })
235
 
236
  return {
237
- # All numeric scores clamped to (0, 1) exclusive — Phase-2 validator requirement
238
  "score": strict_round_clamp(f1, 2),
239
  "precision": strict_round_clamp(precision, 2),
240
  "recall": strict_round_clamp(recall, 2),
 
20
 
21
 
22
  # ---------------------------------------------------------------------------
23
+ # Numerical stability: expose score-like fields inside an open interval.
24
+ # We apply: final_score = clamp(round(raw_score, N)).
25
  # ---------------------------------------------------------------------------
26
  _SCORE_EPSILON = 0.01
27
 
28
  def _clamp_score(score: float) -> float:
29
+ """Keep a score inside the open interval [0.01, 0.99]."""
30
  if score <= 0.01:
31
  return 0.01
32
  elif score >= 0.99:
 
34
  return score
35
 
36
  def strict_round_clamp(raw_score: float, n_digits: int = 2) -> float:
37
+ """Round first, then keep the result inside [0.01, 0.99]."""
38
  rounded = round(raw_score, n_digits)
39
  if rounded <= 0.01:
40
  return 0.01
 
56
  - score: float (0, 1) exclusive (unweighted F1 — primary metric)
57
  - weighted_score: float (0, 1) exclusive (severity-weighted F1)
58
  - partial_credit_score: float (0, 1) exclusive (with partial credit)
59
+ - precision/recall: standard metrics, bounded to (0, 1)
60
  - true_positives/false_positives/false_negatives: counts
61
  - total_findings/total_errors: counts
62
  - matched_errors/missed_errors/false_positive_list: details
 
234
  })
235
 
236
  return {
237
+ # Keep public score-like outputs in a bounded open interval.
238
  "score": strict_round_clamp(f1, 2),
239
  "precision": strict_round_clamp(precision, 2),
240
  "recall": strict_round_clamp(recall, 2),
inference.py CHANGED
@@ -65,7 +65,7 @@ SUCCESS_SCORE_THRESHOLD = 0.5
65
 
66
 
67
  def strict_unit_interval(value: Any, default: float = 0.01) -> float:
68
- """Return a finite float strictly within (0, 1) using contest-safe bounds."""
69
  try:
70
  num = float(value)
71
  except (TypeError, ValueError):
 
65
 
66
 
67
  def strict_unit_interval(value: Any, default: float = 0.01) -> float:
68
+ """Return a finite float constrained to a stable open interval."""
69
  try:
70
  num = float(value)
71
  except (TypeError, ValueError):
server/app.py CHANGED
@@ -305,13 +305,13 @@ async def get_grader_score(session_id: Optional[str] = None):
305
  }
306
 
307
  def final_clamp(val: float) -> float:
308
- """Ultimate pit stop: guarantees NO score is ever less than or equal to 0, or greater than or equal to 1."""
309
  return max(0.01, min(0.99, val))
310
 
311
  return {
312
  "status": "completed",
313
  "task_id": env.state.task_id,
314
- # Primary score (final pit stop applied)
315
  "score": final_clamp(result["score"]),
316
  "precision": final_clamp(result["precision"]),
317
  "recall": final_clamp(result["recall"]),
@@ -319,7 +319,7 @@ async def get_grader_score(session_id: Optional[str] = None):
319
  "false_positives": result["false_positives"],
320
  "false_negatives": result["false_negatives"],
321
  "total_errors": result["total_errors"],
322
- # Enhanced scoring with final clamp
323
  "weighted_score": final_clamp(result.get("weighted_score", result["score"])),
324
  "partial_credit_score": final_clamp(result.get("partial_credit_score", result["score"])),
325
  "partial_matches": result.get("partial_matches", 0),
 
305
  }
306
 
307
  def final_clamp(val: float) -> float:
308
+ """Keep score-like fields within a stable open interval."""
309
  return max(0.01, min(0.99, val))
310
 
311
  return {
312
  "status": "completed",
313
  "task_id": env.state.task_id,
314
+ # Primary score fields with a final stability clamp.
315
  "score": final_clamp(result["score"]),
316
  "precision": final_clamp(result["precision"]),
317
  "recall": final_clamp(result["recall"]),
 
319
  "false_positives": result["false_positives"],
320
  "false_negatives": result["false_negatives"],
321
  "total_errors": result["total_errors"],
322
+ # Enhanced scoring fields with the same stability clamp.
323
  "weighted_score": final_clamp(result.get("weighted_score", result["score"])),
324
  "partial_credit_score": final_clamp(result.get("partial_credit_score", result["score"])),
325
  "partial_matches": result.get("partial_matches", 0),