import json from typing import Dict, Any, List def grade(output: Dict[str, Any], rubric_path: str) -> Dict[str, Any]: with open(rubric_path, "r", encoding="utf-8") as f: rubric = json.load(f) score = 0 max_score = 0 notes: List[str] = [] # Exact-set checks for check in rubric.get("set_equals", []): path = check["path"] # e.g., "prioritization.groups_prioritized" expected = set(check["expected"]) actual = get_path(output, path) max_score += 1 if set(actual) == expected: score += 1 else: notes.append(f"Set mismatch at {path}. Expected {list(expected)} got {actual}") # Must-contain strings for check in rubric.get("must_contain", []): path = check["path"] expected_substr = check["substr"].lower() actual = str(get_path(output, path)).lower() max_score += 1 if expected_substr in actual: score += 1 else: notes.append(f"Missing '{check['substr']}' in {path}") # Numeric equals (tolerance) for check in rubric.get("numeric_equals", []): path = check["path"] expected = float(check["expected"]) tol = float(check.get("tolerance", 0.0)) actual = float(get_path(output, path)) max_score += 1 if abs(actual - expected) <= tol: score += 1 else: notes.append(f"Numeric diff at {path}: expected {expected}±{tol}, got {actual}") return {"score": score, "max_score": max_score, "notes": notes} def get_path(obj, path: str): cur = obj for part in path.split("."): cur = cur[part] return cur