"""Pure grading functions — no I/O, no global state.""" from __future__ import annotations import re def _keyword_found(keyword: str, text: str) -> bool: """Case-insensitive search. Uses word boundaries for alphanumeric keywords to avoid substring false positives (e.g. 'null' matching 'nullable').""" kw = keyword.lower() text = text.lower() if kw and re.match(r"\w", kw[0]) and re.match(r"\w", kw[-1]): return bool(re.search(r"\b" + re.escape(kw) + r"\b", text)) return kw in text def check_comment(comment: str, bugs: list) -> list[int]: """Return indices of bugs matched by this comment (for step-level rewards).""" text = comment.lower() matched: list[int] = [] for i, keyword_list in enumerate(bugs): if isinstance(keyword_list, str): keyword_list = [keyword_list] if any(_keyword_found(kw, text) for kw in keyword_list): matched.append(i) return matched def grade(ground_truth: dict, comments: list[str], decision: str) -> dict: """Score a completed review session against ground truth. Returns score strictly in (0, 1) to satisfy OpenEnv validation constraints. """ full_text = " ".join(comments).lower() bugs: list = ground_truth.get("bugs", []) should_approve: bool = ground_truth.get("should_approve", False) bug_breakdown = [] bugs_found = 0 for keyword_list in bugs: if isinstance(keyword_list, str): keyword_list = [keyword_list] matched_kw = next((kw for kw in keyword_list if _keyword_found(kw, full_text)), None) found = matched_kw is not None if found: bugs_found += 1 bug_breakdown.append({"keywords": keyword_list, "found": found, "matched_by": matched_kw}) total_bugs = len(bugs) bug_detection_rate = bugs_found / total_bugs if total_bugs > 0 else 1.0 decision_correct = (decision == "approve") == should_approve decision_score = 1.0 if decision_correct else 0.0 false_rejection = should_approve and decision == "reject" false_rejection_penalty = -0.2 if false_rejection else 0.0 raw_score = bug_detection_rate * 0.7 + decision_score * 0.3 + false_rejection_penalty final_score = round(max(0.02, min(0.98, raw_score)), 4) # Clamp all float fields so no response value is exactly 0.0 or 1.0 clamped_bug_detection_rate = round(max(0.02, min(0.98, bug_detection_rate)), 4) clamped_decision_score = round(max(0.02, min(0.98, decision_score)), 4) clamped_penalty = round(max(-0.98, min(-0.02, false_rejection_penalty)) if false_rejection else 0.02, 4) return { "score": final_score, "bug_detection_rate": clamped_bug_detection_rate, "bugs_found": bugs_found, "total_bugs": total_bugs, "decision_correct": decision_correct, "decision_score": clamped_decision_score, "false_rejection_penalty": clamped_penalty, "bug_breakdown": bug_breakdown, }