""" grader.py — Reward function for Task 1: Prompt Sensitivity Detection Scoring (total 1.0): 0.50 — verdict correctness 0.25 — sensitive_variant_index accuracy 0.15 — confidence calibration 0.10 — explanation quality """ from __future__ import annotations from typing import Any, Dict, Tuple def grade_sensitivity(action: Any, scenario: Dict) -> Tuple[float, Dict]: verdict = str(getattr(action, "verdict", "")).strip().lower() confidence = float(getattr(action, "confidence", 0.0)) explanation = str(getattr(action, "explanation", "")) sens_idx = getattr(action, "sensitive_variant_index", None) ground_truth = scenario["ground_truth"] gt_index = scenario["sensitive_index"] reward = 0.0 breakdown = {} # 1. Verdict (0.50) if verdict == ground_truth: reward += 0.50 breakdown["verdict"] = 0.50 elif verdict == "partial" and ground_truth == "sensitive": reward += 0.20 breakdown["verdict"] = 0.20 else: breakdown["verdict"] = 0.0 # 2. Sensitive index (0.25) if ground_truth == "sensitive": if sens_idx is not None and int(sens_idx) == gt_index: reward += 0.25 breakdown["sensitive_index"] = 0.25 else: breakdown["sensitive_index"] = 0.0 else: if sens_idx is None or int(sens_idx) == -1: reward += 0.25 breakdown["sensitive_index"] = 0.25 else: breakdown["sensitive_index"] = 0.0 # 3. Confidence calibration (0.15) confidence = max(0.0, min(1.0, confidence)) verdict_correct = breakdown["verdict"] >= 0.50 if verdict_correct: reward += 0.15 if confidence >= 0.6 else round(confidence / 0.6 * 0.15, 3) breakdown["confidence"] = 0.15 if confidence >= 0.6 else round(confidence / 0.6 * 0.15, 3) else: penalty = -0.05 if confidence > 0.8 else 0.0 reward += penalty breakdown["confidence"] = penalty # 4. Explanation quality (0.10) if len(explanation.strip()) >= 20: reward += 0.10 breakdown["explanation"] = 0.10 else: breakdown["explanation"] = 0.0 reward = round(max(0.0, min(1.0, reward)), 4) return reward, {"reward": reward, "breakdown": breakdown}