Spaces:
Running
Running
Commit ·
f76bbd6
1
Parent(s): 4943c0a
Refine comment wording for scoring stability paths
Browse files- financial_audit_env/server/app.py +3 -3
- financial_audit_env/server/environment.py +3 -4
- financial_audit_env/server/graders.py +6 -6
- inference.py +1 -1
- server/app.py +3 -3
financial_audit_env/server/app.py
CHANGED
|
@@ -305,13 +305,13 @@ async def get_grader_score(session_id: Optional[str] = None):
|
|
| 305 |
}
|
| 306 |
|
| 307 |
def final_clamp(val: float) -> float:
|
| 308 |
-
"""
|
| 309 |
return max(0.01, min(0.99, val))
|
| 310 |
|
| 311 |
return {
|
| 312 |
"status": "completed",
|
| 313 |
"task_id": env.state.task_id,
|
| 314 |
-
# Primary score fields with final
|
| 315 |
"score": final_clamp(result["score"]),
|
| 316 |
"precision": final_clamp(result["precision"]),
|
| 317 |
"recall": final_clamp(result["recall"]),
|
|
@@ -319,7 +319,7 @@ async def get_grader_score(session_id: Optional[str] = None):
|
|
| 319 |
"false_positives": result["false_positives"],
|
| 320 |
"false_negatives": result["false_negatives"],
|
| 321 |
"total_errors": result["total_errors"],
|
| 322 |
-
# Enhanced scoring fields with
|
| 323 |
"weighted_score": final_clamp(result.get("weighted_score", result["score"])),
|
| 324 |
"partial_credit_score": final_clamp(result.get("partial_credit_score", result["score"])),
|
| 325 |
"partial_matches": result.get("partial_matches", 0),
|
|
|
|
| 305 |
}
|
| 306 |
|
| 307 |
def final_clamp(val: float) -> float:
|
| 308 |
+
"""Keep score-like fields within a stable open interval."""
|
| 309 |
return max(0.01, min(0.99, val))
|
| 310 |
|
| 311 |
return {
|
| 312 |
"status": "completed",
|
| 313 |
"task_id": env.state.task_id,
|
| 314 |
+
# Primary score fields with a final stability clamp.
|
| 315 |
"score": final_clamp(result["score"]),
|
| 316 |
"precision": final_clamp(result["precision"]),
|
| 317 |
"recall": final_clamp(result["recall"]),
|
|
|
|
| 319 |
"false_positives": result["false_positives"],
|
| 320 |
"false_negatives": result["false_negatives"],
|
| 321 |
"total_errors": result["total_errors"],
|
| 322 |
+
# Enhanced scoring fields with the same stability clamp.
|
| 323 |
"weighted_score": final_clamp(result.get("weighted_score", result["score"])),
|
| 324 |
"partial_credit_score": final_clamp(result.get("partial_credit_score", result["score"])),
|
| 325 |
"partial_matches": result.get("partial_matches", 0),
|
financial_audit_env/server/environment.py
CHANGED
|
@@ -234,14 +234,13 @@ class FinancialAuditEnvironment(Environment):
|
|
| 234 |
# Check if episode should end
|
| 235 |
is_final = action.submit_final or step_num >= self._task["max_steps"]
|
| 236 |
|
| 237 |
-
# Compute step reward
|
| 238 |
-
#
|
| 239 |
-
# Otherwise, the evaluator's `sum(rewards)` will exceed 1.0.
|
| 240 |
if not is_final:
|
| 241 |
step_reward = 0.0
|
| 242 |
else:
|
| 243 |
final_grader = compute_f1_score(self._findings, self._ground_truth)
|
| 244 |
-
#
|
| 245 |
step_reward = max(0.01, min(0.99, final_grader["score"]))
|
| 246 |
|
| 247 |
self._episode_reward += step_reward
|
|
|
|
| 234 |
# Check if episode should end
|
| 235 |
is_final = action.submit_final or step_num >= self._task["max_steps"]
|
| 236 |
|
| 237 |
+
# Compute step reward.
|
| 238 |
+
# Keep cumulative episode rewards aligned with the final task score.
|
|
|
|
| 239 |
if not is_final:
|
| 240 |
step_reward = 0.0
|
| 241 |
else:
|
| 242 |
final_grader = compute_f1_score(self._findings, self._ground_truth)
|
| 243 |
+
# Use the bounded final score as the terminal reward.
|
| 244 |
step_reward = max(0.01, min(0.99, final_grader["score"]))
|
| 245 |
|
| 246 |
self._episode_reward += step_reward
|
financial_audit_env/server/graders.py
CHANGED
|
@@ -20,13 +20,13 @@ from .data_generator import ERROR_MONETARY_VALUES, ERROR_SEVERITY_WEIGHTS
|
|
| 20 |
|
| 21 |
|
| 22 |
# ---------------------------------------------------------------------------
|
| 23 |
-
#
|
| 24 |
-
# We
|
| 25 |
# ---------------------------------------------------------------------------
|
| 26 |
_SCORE_EPSILON = 0.01
|
| 27 |
|
| 28 |
def _clamp_score(score: float) -> float:
|
| 29 |
-
"""
|
| 30 |
if score <= 0.01:
|
| 31 |
return 0.01
|
| 32 |
elif score >= 0.99:
|
|
@@ -34,7 +34,7 @@ def _clamp_score(score: float) -> float:
|
|
| 34 |
return score
|
| 35 |
|
| 36 |
def strict_round_clamp(raw_score: float, n_digits: int = 2) -> float:
|
| 37 |
-
"""
|
| 38 |
rounded = round(raw_score, n_digits)
|
| 39 |
if rounded <= 0.01:
|
| 40 |
return 0.01
|
|
@@ -56,7 +56,7 @@ def compute_f1_score(
|
|
| 56 |
- score: float (0, 1) exclusive (unweighted F1 — primary metric)
|
| 57 |
- weighted_score: float (0, 1) exclusive (severity-weighted F1)
|
| 58 |
- partial_credit_score: float (0, 1) exclusive (with partial credit)
|
| 59 |
-
- precision/recall: standard metrics,
|
| 60 |
- true_positives/false_positives/false_negatives: counts
|
| 61 |
- total_findings/total_errors: counts
|
| 62 |
- matched_errors/missed_errors/false_positive_list: details
|
|
@@ -234,7 +234,7 @@ def compute_f1_score(
|
|
| 234 |
})
|
| 235 |
|
| 236 |
return {
|
| 237 |
-
#
|
| 238 |
"score": strict_round_clamp(f1, 2),
|
| 239 |
"precision": strict_round_clamp(precision, 2),
|
| 240 |
"recall": strict_round_clamp(recall, 2),
|
|
|
|
| 20 |
|
| 21 |
|
| 22 |
# ---------------------------------------------------------------------------
|
| 23 |
+
# Numerical stability: expose score-like fields inside an open interval.
|
| 24 |
+
# We apply: final_score = clamp(round(raw_score, N)).
|
| 25 |
# ---------------------------------------------------------------------------
|
| 26 |
_SCORE_EPSILON = 0.01
|
| 27 |
|
| 28 |
def _clamp_score(score: float) -> float:
|
| 29 |
+
"""Keep a score inside the open interval [0.01, 0.99]."""
|
| 30 |
if score <= 0.01:
|
| 31 |
return 0.01
|
| 32 |
elif score >= 0.99:
|
|
|
|
| 34 |
return score
|
| 35 |
|
| 36 |
def strict_round_clamp(raw_score: float, n_digits: int = 2) -> float:
|
| 37 |
+
"""Round first, then keep the result inside [0.01, 0.99]."""
|
| 38 |
rounded = round(raw_score, n_digits)
|
| 39 |
if rounded <= 0.01:
|
| 40 |
return 0.01
|
|
|
|
| 56 |
- score: float (0, 1) exclusive (unweighted F1 — primary metric)
|
| 57 |
- weighted_score: float (0, 1) exclusive (severity-weighted F1)
|
| 58 |
- partial_credit_score: float (0, 1) exclusive (with partial credit)
|
| 59 |
+
- precision/recall: standard metrics, bounded to (0, 1)
|
| 60 |
- true_positives/false_positives/false_negatives: counts
|
| 61 |
- total_findings/total_errors: counts
|
| 62 |
- matched_errors/missed_errors/false_positive_list: details
|
|
|
|
| 234 |
})
|
| 235 |
|
| 236 |
return {
|
| 237 |
+
# Keep public score-like outputs in a bounded open interval.
|
| 238 |
"score": strict_round_clamp(f1, 2),
|
| 239 |
"precision": strict_round_clamp(precision, 2),
|
| 240 |
"recall": strict_round_clamp(recall, 2),
|
inference.py
CHANGED
|
@@ -65,7 +65,7 @@ SUCCESS_SCORE_THRESHOLD = 0.5
|
|
| 65 |
|
| 66 |
|
| 67 |
def strict_unit_interval(value: Any, default: float = 0.01) -> float:
|
| 68 |
-
"""Return a finite float
|
| 69 |
try:
|
| 70 |
num = float(value)
|
| 71 |
except (TypeError, ValueError):
|
|
|
|
| 65 |
|
| 66 |
|
| 67 |
def strict_unit_interval(value: Any, default: float = 0.01) -> float:
|
| 68 |
+
"""Return a finite float constrained to a stable open interval."""
|
| 69 |
try:
|
| 70 |
num = float(value)
|
| 71 |
except (TypeError, ValueError):
|
server/app.py
CHANGED
|
@@ -305,13 +305,13 @@ async def get_grader_score(session_id: Optional[str] = None):
|
|
| 305 |
}
|
| 306 |
|
| 307 |
def final_clamp(val: float) -> float:
|
| 308 |
-
"""
|
| 309 |
return max(0.01, min(0.99, val))
|
| 310 |
|
| 311 |
return {
|
| 312 |
"status": "completed",
|
| 313 |
"task_id": env.state.task_id,
|
| 314 |
-
# Primary score
|
| 315 |
"score": final_clamp(result["score"]),
|
| 316 |
"precision": final_clamp(result["precision"]),
|
| 317 |
"recall": final_clamp(result["recall"]),
|
|
@@ -319,7 +319,7 @@ async def get_grader_score(session_id: Optional[str] = None):
|
|
| 319 |
"false_positives": result["false_positives"],
|
| 320 |
"false_negatives": result["false_negatives"],
|
| 321 |
"total_errors": result["total_errors"],
|
| 322 |
-
# Enhanced scoring with
|
| 323 |
"weighted_score": final_clamp(result.get("weighted_score", result["score"])),
|
| 324 |
"partial_credit_score": final_clamp(result.get("partial_credit_score", result["score"])),
|
| 325 |
"partial_matches": result.get("partial_matches", 0),
|
|
|
|
| 305 |
}
|
| 306 |
|
| 307 |
def final_clamp(val: float) -> float:
|
| 308 |
+
"""Keep score-like fields within a stable open interval."""
|
| 309 |
return max(0.01, min(0.99, val))
|
| 310 |
|
| 311 |
return {
|
| 312 |
"status": "completed",
|
| 313 |
"task_id": env.state.task_id,
|
| 314 |
+
# Primary score fields with a final stability clamp.
|
| 315 |
"score": final_clamp(result["score"]),
|
| 316 |
"precision": final_clamp(result["precision"]),
|
| 317 |
"recall": final_clamp(result["recall"]),
|
|
|
|
| 319 |
"false_positives": result["false_positives"],
|
| 320 |
"false_negatives": result["false_negatives"],
|
| 321 |
"total_errors": result["total_errors"],
|
| 322 |
+
# Enhanced scoring fields with the same stability clamp.
|
| 323 |
"weighted_score": final_clamp(result.get("weighted_score", result["score"])),
|
| 324 |
"partial_credit_score": final_clamp(result.get("partial_credit_score", result["score"])),
|
| 325 |
"partial_matches": result.get("partial_matches", 0),
|