""" grader.py (Task 3 – Rule Checker) ------------------------------------ Deterministic grader for function-identification submissions. Score table ─────────── 1.0 → submitted function is the exact target (case-insensitive) 0.3 → submitted function is a direct internal subfunction of the target (a contract-internal function called by the target in the call graph) 0.0 → anything else Reward table (ONE submission per episode) score 1.0 → +5.0 score 0.3 → +1.5 score 0.0 → -1.5 """ from __future__ import annotations import json from typing import Dict, Any class Task3Grader: """ Grades a Task 3 submit_function submission. Parameters ---------- target_function : exact name of the rule-breaking function partial_credit_functions: list of internal functions that get partial credit (direct callees of the target that are contract functions) """ SCORE_CORRECT = 1.0 SCORE_PARTIAL = 0.3 SCORE_WRONG = 0.0 REWARD_CORRECT = 5.0 REWARD_PARTIAL = 1.5 REWARD_WRONG = -1.5 def __init__(self, target_function: Dict[str, Any], property_specification: Dict | str) -> None: self.target_function = target_function self.property_specification = property_specification def grade(self, submitted_function: str) -> float: """Returns deterministic score in {0.0, 0.3, 1.0}.""" norm = submitted_function.strip().lower() if norm == self.target_function["name"].strip().lower(): return self.SCORE_CORRECT if norm in self.target_function.get("code", "").strip().lower(): return self.SCORE_PARTIAL return self.SCORE_WRONG def reward_for_score(self, score: float) -> float: """Maps score → terminal reward.""" if score >= 0.9: return self.REWARD_CORRECT if score >= 0.2: return self.REWARD_PARTIAL return self.REWARD_WRONG def grade_and_reward(self, submitted_function: str): """Convenience: returns (score, reward).""" score = self.grade(submitted_function) return score, self.reward_for_score(score) def get_canonical_answer(self) -> Dict[str, Dict | str]: """For debugging / logging only — do not expose to the agent.""" return { "target_function": self.target_function, "property_specification": json.dumps(self.property_specification) if isinstance(self.property_specification, dict) else self.property_specification, }