Spaces:
Sleeping
Sleeping
File size: 1,689 Bytes
5cf4edb |
1 2 3 4 5 6 7 8 9 10 11 12 13 14 15 16 17 18 19 20 21 22 23 24 25 26 27 28 29 30 31 32 33 34 35 36 37 38 39 40 41 42 43 44 45 46 47 48 49 50 51 52 53 |
import json
from typing import Dict, Any, List
def grade(output: Dict[str, Any], rubric_path: str) -> Dict[str, Any]:
with open(rubric_path, "r", encoding="utf-8") as f:
rubric = json.load(f)
score = 0
max_score = 0
notes: List[str] = []
# Exact-set checks
for check in rubric.get("set_equals", []):
path = check["path"] # e.g., "prioritization.groups_prioritized"
expected = set(check["expected"])
actual = get_path(output, path)
max_score += 1
if set(actual) == expected:
score += 1
else:
notes.append(f"Set mismatch at {path}. Expected {list(expected)} got {actual}")
# Must-contain strings
for check in rubric.get("must_contain", []):
path = check["path"]
expected_substr = check["substr"].lower()
actual = str(get_path(output, path)).lower()
max_score += 1
if expected_substr in actual:
score += 1
else:
notes.append(f"Missing '{check['substr']}' in {path}")
# Numeric equals (tolerance)
for check in rubric.get("numeric_equals", []):
path = check["path"]
expected = float(check["expected"])
tol = float(check.get("tolerance", 0.0))
actual = float(get_path(output, path))
max_score += 1
if abs(actual - expected) <= tol:
score += 1
else:
notes.append(f"Numeric diff at {path}: expected {expected}±{tol}, got {actual}")
return {"score": score, "max_score": max_score, "notes": notes}
def get_path(obj, path: str):
cur = obj
for part in path.split("."):
cur = cur[part]
return cur
|