server/grader.py

"""
grader.py — Reward function for Task 1: Prompt Sensitivity Detection
Scoring (total 1.0):
  0.50 — verdict correctness
  0.25 — sensitive_variant_index accuracy
  0.15 — confidence calibration
  0.10 — explanation quality
"""
from __future__ import annotations
from typing import Any, Dict, Tuple


def grade_sensitivity(action: Any, scenario: Dict) -> Tuple[float, Dict]:
    verdict    = str(getattr(action, "verdict", "")).strip().lower()
    confidence = float(getattr(action, "confidence", 0.0))
    explanation = str(getattr(action, "explanation", ""))
    sens_idx   = getattr(action, "sensitive_variant_index", None)

    ground_truth = scenario["ground_truth"]
    gt_index     = scenario["sensitive_index"]

    reward = 0.0
    breakdown = {}

    # 1. Verdict (0.50)
    if verdict == ground_truth:
        reward += 0.50
        breakdown["verdict"] = 0.50
    elif verdict == "partial" and ground_truth == "sensitive":
        reward += 0.20
        breakdown["verdict"] = 0.20
    else:
        breakdown["verdict"] = 0.0

    # 2. Sensitive index (0.25)
    if ground_truth == "sensitive":
        if sens_idx is not None and int(sens_idx) == gt_index:
            reward += 0.25
            breakdown["sensitive_index"] = 0.25
        else:
            breakdown["sensitive_index"] = 0.0
    else:
        if sens_idx is None or int(sens_idx) == -1:
            reward += 0.25
            breakdown["sensitive_index"] = 0.25
        else:
            breakdown["sensitive_index"] = 0.0

    # 3. Confidence calibration (0.15)
    confidence = max(0.0, min(1.0, confidence))
    verdict_correct = breakdown["verdict"] >= 0.50
    if verdict_correct:
        reward += 0.15 if confidence >= 0.6 else round(confidence / 0.6 * 0.15, 3)
        breakdown["confidence"] = 0.15 if confidence >= 0.6 else round(confidence / 0.6 * 0.15, 3)
    else:
        penalty = -0.05 if confidence > 0.8 else 0.0
        reward += penalty
        breakdown["confidence"] = penalty

    # 4. Explanation quality (0.10)
    if len(explanation.strip()) >= 20:
        reward += 0.10
        breakdown["explanation"] = 0.10
    else:
        breakdown["explanation"] = 0.0

    reward = round(max(0.0, min(1.0, reward)), 4)
    return reward, {"reward": reward, "breakdown": breakdown}