Spaces:
Running
Running
File size: 2,636 Bytes
7203787 1b91307 7203787 1b91307 7203787 1b91307 7203787 1b91307 7203787 1b91307 7203787 1b91307 7203787 | 1 2 3 4 5 6 7 8 9 10 11 12 13 14 15 16 17 18 19 20 21 22 23 24 25 26 27 28 29 30 31 32 33 34 35 36 37 38 39 40 41 42 43 44 45 46 47 48 49 50 51 52 53 54 55 56 57 58 59 60 61 62 63 64 65 66 67 68 69 70 71 72 73 74 75 76 | """
grader.py (Task 3 – Rule Checker)
------------------------------------
Deterministic grader for function-identification submissions.
Score table
───────────
1.0 → submitted function is the exact target (case-insensitive)
0.3 → submitted function is a direct internal subfunction of the target
(a contract-internal function called by the target in the call graph)
0.0 → anything else
Reward table (ONE submission per episode)
score 1.0 → +5.0
score 0.3 → +1.5
score 0.0 → -1.5
"""
from __future__ import annotations
import json
from typing import Dict, Any
class Task3Grader:
"""
Grades a Task 3 submit_function submission.
Parameters
----------
target_function : exact name of the rule-breaking function
partial_credit_functions: list of internal functions that get partial credit
(direct callees of the target that are contract functions)
"""
SCORE_CORRECT = 1.0
SCORE_PARTIAL = 0.3
SCORE_WRONG = 0.0
REWARD_CORRECT = 5.0
REWARD_PARTIAL = 1.5
REWARD_WRONG = -1.5
def __init__(self, target_function: Dict[str, Any], property_specification: Dict | str) -> None:
self.target_function = target_function
self.property_specification = property_specification
def grade(self, submitted_function: str) -> float:
"""Returns deterministic score in {0.0, 0.3, 1.0}."""
norm = submitted_function.strip().lower()
if norm == self.target_function["name"].strip().lower():
return self.SCORE_CORRECT
if norm in self.target_function.get("code", "").strip().lower():
return self.SCORE_PARTIAL
return self.SCORE_WRONG
def reward_for_score(self, score: float) -> float:
"""Maps score → terminal reward."""
if score >= 0.9:
return self.REWARD_CORRECT
if score >= 0.2:
return self.REWARD_PARTIAL
return self.REWARD_WRONG
def grade_and_reward(self, submitted_function: str):
"""Convenience: returns (score, reward)."""
score = self.grade(submitted_function)
return score, self.reward_for_score(score)
def get_canonical_answer(self) -> Dict[str, Dict | str]:
"""For debugging / logging only — do not expose to the agent."""
return {
"target_function": self.target_function,
"property_specification": json.dumps(self.property_specification)
if isinstance(self.property_specification, dict) else self.property_specification,
}
|