graders/rule_grader.py

import json
from typing import Dict, Any, List

def grade(output: Dict[str, Any], rubric_path: str) -> Dict[str, Any]:
    with open(rubric_path, "r", encoding="utf-8") as f:
        rubric = json.load(f)

    score = 0
    max_score = 0
    notes: List[str] = []

    # Exact-set checks
    for check in rubric.get("set_equals", []):
        path = check["path"]  # e.g., "prioritization.groups_prioritized"
        expected = set(check["expected"])
        actual = get_path(output, path)
        max_score += 1
        if set(actual) == expected:
            score += 1
        else:
            notes.append(f"Set mismatch at {path}. Expected {list(expected)} got {actual}")

    # Must-contain strings
    for check in rubric.get("must_contain", []):
        path = check["path"]
        expected_substr = check["substr"].lower()
        actual = str(get_path(output, path)).lower()
        max_score += 1
        if expected_substr in actual:
            score += 1
        else:
            notes.append(f"Missing '{check['substr']}' in {path}")

    # Numeric equals (tolerance)
    for check in rubric.get("numeric_equals", []):
        path = check["path"]
        expected = float(check["expected"])
        tol = float(check.get("tolerance", 0.0))
        actual = float(get_path(output, path))
        max_score += 1
        if abs(actual - expected) <= tol:
            score += 1
        else:
            notes.append(f"Numeric diff at {path}: expected {expected}±{tol}, got {actual}")

    return {"score": score, "max_score": max_score, "notes": notes}

def get_path(obj, path: str):
    cur = obj
    for part in path.split("."):
        cur = cur[part]
    return cur