Create grader.py

grader.py

@@ -0,0 +1,25 @@
+def grade_comment(comment: str, expected_keywords: list, expert_comment: str) -> float:
+    """
+    Returns a score in [0, 1] based on keyword coverage and some simple heuristics.
+    """
+    comment_lower = comment.lower()
+    # 1. Keyword coverage (primary)
+    matched = sum(1 for kw in expected_keywords if kw in comment_lower)
+    kw_score = min(1.0, matched / max(1, len(expected_keywords) // 2))  # partial if at least half
+
+    # 2. Bonus for reasonable length (≥ 15 words)
+    words = comment.split()
+    length_bonus = 0.1 if len(words) >= 15 else 0.0
+
+    # 3. Penalty if the comment contains "skip" or "done" (agent didn't really comment)
+    if any(x in comment_lower for x in ["skip", "done", "no comment"]):
+        return 0.0
+
+    # 4. Penalty for extremely short or generic comments
+    if len(words) < 5 or comment_lower in ["lgtm", "looks good", "good"]:
+        penalty = 0.2
+    else:
+        penalty = 0.0
+
+    final_score = kw_score + length_bonus - penalty
+    return max(0.0, min(1.0, final_score))