Spaces:
Running
Running
| """ | |
| grader.py (Task 3 – Rule Checker) | |
| ------------------------------------ | |
| Deterministic grader for function-identification submissions. | |
| Score table | |
| ─────────── | |
| 1.0 → submitted function is the exact target (case-insensitive) | |
| 0.3 → submitted function is a direct internal subfunction of the target | |
| (a contract-internal function called by the target in the call graph) | |
| 0.0 → anything else | |
| Reward table (ONE submission per episode) | |
| score 1.0 → +5.0 | |
| score 0.3 → +1.5 | |
| score 0.0 → -1.5 | |
| """ | |
| from __future__ import annotations | |
| import json | |
| from typing import Dict, Any | |
| class Task3Grader: | |
| """ | |
| Grades a Task 3 submit_function submission. | |
| Parameters | |
| ---------- | |
| target_function : exact name of the rule-breaking function | |
| partial_credit_functions: list of internal functions that get partial credit | |
| (direct callees of the target that are contract functions) | |
| """ | |
| SCORE_CORRECT = 1.0 | |
| SCORE_PARTIAL = 0.3 | |
| SCORE_WRONG = 0.0 | |
| REWARD_CORRECT = 5.0 | |
| REWARD_PARTIAL = 1.5 | |
| REWARD_WRONG = -1.5 | |
| def __init__(self, target_function: Dict[str, Any], property_specification: Dict | str) -> None: | |
| self.target_function = target_function | |
| self.property_specification = property_specification | |
| def grade(self, submitted_function: str) -> float: | |
| """Returns deterministic score in {0.0, 0.3, 1.0}.""" | |
| norm = submitted_function.strip().lower() | |
| if norm == self.target_function["name"].strip().lower(): | |
| return self.SCORE_CORRECT | |
| if norm in self.target_function.get("code", "").strip().lower(): | |
| return self.SCORE_PARTIAL | |
| return self.SCORE_WRONG | |
| def reward_for_score(self, score: float) -> float: | |
| """Maps score → terminal reward.""" | |
| if score >= 0.9: | |
| return self.REWARD_CORRECT | |
| if score >= 0.2: | |
| return self.REWARD_PARTIAL | |
| return self.REWARD_WRONG | |
| def grade_and_reward(self, submitted_function: str): | |
| """Convenience: returns (score, reward).""" | |
| score = self.grade(submitted_function) | |
| return score, self.reward_for_score(score) | |
| def get_canonical_answer(self) -> Dict[str, Dict | str]: | |
| """For debugging / logging only — do not expose to the agent.""" | |
| return { | |
| "target_function": self.target_function, | |
| "property_specification": json.dumps(self.property_specification) | |
| if isinstance(self.property_specification, dict) else self.property_specification, | |
| } | |