Spaces:
Sleeping
Sleeping
File size: 2,984 Bytes
8d96200 80454a1 8d96200 7e9c2fa 8d96200 80454a1 cdd4363 8d96200 56ac1f3 8d96200 56ac1f3 8d96200 56ac1f3 8d96200 | 1 2 3 4 5 6 7 8 9 10 11 12 13 14 15 16 17 18 19 20 21 22 23 24 25 26 27 28 29 30 31 32 33 34 35 36 37 38 39 40 41 42 43 44 45 46 47 48 49 50 51 52 53 54 55 56 57 58 59 60 61 62 63 64 65 66 67 68 69 70 71 72 | """Pure grading functions — no I/O, no global state."""
from __future__ import annotations
import re
def _keyword_found(keyword: str, text: str) -> bool:
"""Case-insensitive search. Uses word boundaries for alphanumeric keywords
to avoid substring false positives (e.g. 'null' matching 'nullable')."""
kw = keyword.lower()
text = text.lower()
if kw and re.match(r"\w", kw[0]) and re.match(r"\w", kw[-1]):
return bool(re.search(r"\b" + re.escape(kw) + r"\b", text))
return kw in text
def check_comment(comment: str, bugs: list) -> list[int]:
"""Return indices of bugs matched by this comment (for step-level rewards)."""
text = comment.lower()
matched: list[int] = []
for i, keyword_list in enumerate(bugs):
if isinstance(keyword_list, str):
keyword_list = [keyword_list]
if any(_keyword_found(kw, text) for kw in keyword_list):
matched.append(i)
return matched
def grade(ground_truth: dict, comments: list[str], decision: str) -> dict:
"""Score a completed review session against ground truth.
Returns score strictly in (0, 1) to satisfy OpenEnv validation constraints.
"""
full_text = " ".join(comments).lower()
bugs: list = ground_truth.get("bugs", [])
should_approve: bool = ground_truth.get("should_approve", False)
bug_breakdown = []
bugs_found = 0
for keyword_list in bugs:
if isinstance(keyword_list, str):
keyword_list = [keyword_list]
matched_kw = next((kw for kw in keyword_list if _keyword_found(kw, full_text)), None)
found = matched_kw is not None
if found:
bugs_found += 1
bug_breakdown.append({"keywords": keyword_list, "found": found, "matched_by": matched_kw})
total_bugs = len(bugs)
bug_detection_rate = bugs_found / total_bugs if total_bugs > 0 else 1.0
decision_correct = (decision == "approve") == should_approve
decision_score = 1.0 if decision_correct else 0.0
false_rejection = should_approve and decision == "reject"
false_rejection_penalty = -0.2 if false_rejection else 0.0
raw_score = bug_detection_rate * 0.7 + decision_score * 0.3 + false_rejection_penalty
final_score = round(max(0.02, min(0.98, raw_score)), 4)
# Clamp all float fields so no response value is exactly 0.0 or 1.0
clamped_bug_detection_rate = round(max(0.02, min(0.98, bug_detection_rate)), 4)
clamped_decision_score = round(max(0.02, min(0.98, decision_score)), 4)
clamped_penalty = round(max(-0.98, min(-0.02, false_rejection_penalty)) if false_rejection else 0.02, 4)
return {
"score": final_score,
"bug_detection_rate": clamped_bug_detection_rate,
"bugs_found": bugs_found,
"total_bugs": total_bugs,
"decision_correct": decision_correct,
"decision_score": clamped_decision_score,
"false_rejection_penalty": clamped_penalty,
"bug_breakdown": bug_breakdown,
}
|