Create graders/rule_grader.py

graders/rule_grader.py

@@ -0,0 +1,52 @@
+import json
+from typing import Dict, Any, List
+
+def grade(output: Dict[str, Any], rubric_path: str) -> Dict[str, Any]:
+    with open(rubric_path, "r", encoding="utf-8") as f:
+        rubric = json.load(f)
+
+    score = 0
+    max_score = 0
+    notes: List[str] = []
+
+    # Exact-set checks
+    for check in rubric.get("set_equals", []):
+        path = check["path"]  # e.g., "prioritization.groups_prioritized"
+        expected = set(check["expected"])
+        actual = get_path(output, path)
+        max_score += 1
+        if set(actual) == expected:
+            score += 1
+        else:
+            notes.append(f"Set mismatch at {path}. Expected {list(expected)} got {actual}")
+
+    # Must-contain strings
+    for check in rubric.get("must_contain", []):
+        path = check["path"]
+        expected_substr = check["substr"].lower()
+        actual = str(get_path(output, path)).lower()
+        max_score += 1
+        if expected_substr in actual:
+            score += 1
+        else:
+            notes.append(f"Missing '{check['substr']}' in {path}")
+
+    # Numeric equals (tolerance)
+    for check in rubric.get("numeric_equals", []):
+        path = check["path"]
+        expected = float(check["expected"])
+        tol = float(check.get("tolerance", 0.0))
+        actual = float(get_path(output, path))
+        max_score += 1
+        if abs(actual - expected) <= tol:
+            score += 1
+        else:
+            notes.append(f"Numeric diff at {path}: expected {expected}±{tol}, got {actual}")
+
+    return {"score": score, "max_score": max_score, "notes": notes}
+
+def get_path(obj, path: str):
+    cur = obj
+    for part in path.split("."):
+        cur = cur[part]
+    return cur