Update grader.py

grader.py

@@ -1,27 +1,152 @@
-def grade_phishing_task(agent_output: str, expected_output: str) -> float:
-    """
-    Evaluates the agent's action against the ground truth.
-    Returns a score between 0.0 and 1.0.
-    """
-    # Normalize inputs to avoid case-sensitivity issues
-    agent_action = agent_output.strip().upper()
-    expected_action = expected_output.strip().upper()
-
-    # Perfect match: Maximum reward
-    if agent_action == expected_action:
-        return 1.0
-    
-    # Partial credit: If agent quarantined instead of blocking (safe but cautious)
-    if agent_action == "QUARANTINE" and expected_action == "BLOCK_DOMAIN":
-        return 0.5
-        
-    # Failure: Incorrect categorization or allowing a threat
-    return 0.0
-
-def calculate_overall_score(task_scores: list) -> float:
-    """
-    Calculates the final reproducible score for the baseline report.
-    """
-    if not task_scores:
-        return 0.0
-    return sum(task_scores) / len(task_scores)
+
+"""
+grader.py – PhishGuard-Env SOC Triage Scoring Logic
+=====================================================
+
+REWARD SCALE CONTRACT
+---------------------
+All rewards are in the OPEN interval (0.0, 1.0).
+The endpoints 0 and 1 are NEVER returned. This is a hard invariant.
+
+Why open-interval?
+  • 1.0 would saturate the leaderboard and imply a theoretically perfect agent.
+  • 0.0 would be indistinguishable from a missing data-point in an RL pipeline.
+  • Every decision carries a non-zero signal so the training gradient never dies.
+
+Full reward table
+-----------------
+  Outcome                                    Reward   Rationale
+  ─────────────────────────────────────────────────────────────────────────────
+  Perfect match (any type)                    0.95    Near-ideal; headroom preserved
+  MALWARE → QUARANTINE (safe containment)     0.75    Correct call, no partial penalty
+  PHISH/BEC → QUARANTINE (under-response)    0.60    Stopped but domain still lives
+  SPAM → BLOCK_DOMAIN (over-escalation)      0.40    Too aggressive, not wrong
+  SPAM → QUARANTINE (lighter escalation)     0.35    Wastes analyst queue
+  General wrong procedure (no harm)          0.10    Wrong but no security/ops impact
+  Business Disruption (SAFE → blocked)       0.05    Ops cost, below drain threshold
+  Security Breach (threat → MARK_SAFE)       0.02    Catastrophic; minimum signal
+
+Health-drain threshold (consumed by env.py)
+-------------------------------------------
+  HEALTH_DRAIN_THRESHOLD = 0.15
+  reward < 0.15  →  lose one life.
+  This covers Security Breach (0.02), Business Disruption (0.05), and
+  General Wrong Procedure (0.10). Cautious/partial-credit scores never
+  drain health, which is the intended design.
+"""
+
+from typing import Tuple
+
+# ── Reward constants ───────────────────────────────────────────────────────────
+# Change values here only — nowhere else in the codebase hard-codes these.
+R_PERFECT               = 0.95
+R_MALWARE_QUARANTINE    = 0.75
+R_PHISH_BEC_QUARANTINE  = 0.60
+R_SPAM_BLOCK            = 0.40
+R_SPAM_QUARANTINE       = 0.35
+R_WRONG_PROCEDURE       = 0.10
+R_DISRUPTION            = 0.05
+R_BREACH                = 0.02
+
+# env.py compares `reward < HEALTH_DRAIN_THRESHOLD` to decide life-loss.
+HEALTH_DRAIN_THRESHOLD  = 0.15
+
+
+def grade_action(
+    agent_output: str,
+    expected_output: str,
+    email_type: str,
+) -> Tuple[float, str]:
+    """
+    Grade one triage decision and return a reward in the open interval (0, 1).
+
+    Parameters
+    ----------
+    agent_output    : Raw action string from the LLM (case-insensitive).
+    expected_output : Ground-truth correct action for this scenario.
+    email_type      : Threat category — PHISH | BEC | MALWARE | SPAM | SAFE.
+
+    Returns
+    -------
+    (reward, message)
+        reward  : float strictly in (0.0, 1.0) — never 0, never 1.
+        message : short verdict string for logs and agent feedback.
+    """
+    agent_action    = agent_output.strip().upper()
+    expected_action = expected_output.strip().upper()
+    etype           = email_type.strip().upper()
+
+    THREAT_TYPES = {"PHISH", "BEC", "MALWARE", "SPAM"}
+
+    # ── 1. PERFECT MATCH ──────────────────────────────────────────────────────
+    if agent_action == expected_action:
+        return R_PERFECT, "PERFECT_TRIAGE: Correct action taken"
+
+    # ── 2. SECURITY BREACH — most severe ──────────────────────────────────────
+    # A dangerous email was rubber-stamped into the inbox.
+    if etype in THREAT_TYPES and agent_action == "MARK_SAFE":
+        return R_BREACH, "SECURITY_BREACH: Threat allowed into the network"
+
+    # ── 3. BUSINESS DISRUPTION — severe false positive ────────────────────────
+    # A clean email was blocked or quarantined.
+    if etype == "SAFE" and agent_action in {"BLOCK_DOMAIN", "QUARANTINE", "MOVE_TO_SPAM"}:
+        return R_DISRUPTION, "BUSINESS_DISRUPTION: Legitimate communication blocked"
+
+    # ── 4. PARTIAL CREDIT — cautious but suboptimal ───────────────────────────
+
+    # MALWARE → QUARANTINE: isolation is the textbook response.
+    # This branch fires only when the scenario's `correct` field differs
+    # (edge case for future scenario expansions — included for robustness).
+    if etype == "MALWARE" and agent_action == "QUARANTINE":
+        return R_MALWARE_QUARANTINE, "CAUTIOUS: Malware isolated — strong containment"
+
+    # PHISH/BEC → QUARANTINE: threat is held but the domain keeps sending.
+    if etype in {"PHISH", "BEC"} and agent_action == "QUARANTINE":
+        return R_PHISH_BEC_QUARANTINE, (
+            "UNDER_RESPONSE: Threat quarantined but domain still active — "
+            "consider BLOCK_DOMAIN"
+        )
+
+    # SPAM → BLOCK_DOMAIN: blocks the domain, but permanently for mere spam.
+    if etype == "SPAM" and agent_action == "BLOCK_DOMAIN":
+        return R_SPAM_BLOCK, (
+            "OVER_ESCALATION: BLOCK_DOMAIN is disproportionate for SPAM — "
+            "prefer MOVE_TO_SPAM"
+        )
+
+    # SPAM → QUARANTINE: lighter over-escalation, still clogs the analyst queue.
+    if etype == "SPAM" and agent_action == "QUARANTINE":
+        return R_SPAM_QUARANTINE, (
+            "OVER_ESCALATION: QUARANTINE wastes analyst capacity for SPAM — "
+            "prefer MOVE_TO_SPAM"
+        )
+
+    # ── 5. GENERAL INCORRECT PROCEDURE ───────────────────────────────────────
+    return R_WRONG_PROCEDURE, "INCORRECT_PROCEDURE: Decision does not match security policy"
+
+
+def calculate_overall_score(task_scores: list) -> float:
+    """
+    Compute the final benchmark score from a list of per-task rewards.
+
+    The result is clamped to (R_BREACH, R_PERFECT) — matching the per-step
+    open-interval contract — so downstream consumers always receive a float
+    that is strictly greater than 0 and strictly less than 1.
+
+    Parameters
+    ----------
+    task_scores : list of floats, each in (0.0, 1.0).
+
+    Returns
+    -------
+    float in (0.0, 1.0) — never exactly 0 or 1.
+    """
+    if not task_scores:
+        # No tasks completed: return the minimum signal value, not zero.
+        return R_BREACH
+
+    raw_avg = sum(task_scores) / len(task_scores)
+
+    # Clamp strictly within the open interval boundaries.
+    clamped = max(R_BREACH, min(R_PERFECT, raw_avg))
+    return round(clamped, 4)