Update grader.py

grader.py

@@ -1,115 +1,113 @@
 """
-grader.py – PhishGuard-Env | Triage Scoring & Task Registry
-==============================================================
-
-REWARD SCALE CONTRACT
-----------------------------------
-All rewards strictly in the open interval (0.0, 1.0) — 0 and 1 never returned.
-
-  Outcome                                    Reward   Rationale
-  ─────────────────────────────────────────────────────────────────────────────
-  Perfect match (any task type)               0.95    Near-ideal; headroom kept
-  MALWARE → QUARANTINE (safe containment)     0.75    Correct containment call
-  PHISH/BEC → QUARANTINE (under-response)    0.60    Held but domain still active
-  SPAM → BLOCK_DOMAIN (over-escalation)      0.40    Too aggressive, not wrong
-  SPAM → QUARANTINE (lighter escalation)     0.35    Wastes analyst queue
-  General wrong procedure (no active harm)   0.10    Wrong but not dangerous
-  Business Disruption (SAFE email blocked)   0.05    Ops impact, below drain floor
-  Security Breach (threat → MARK_SAFE)       0.02    Catastrophic; minimum signal
-
-Health-drain threshold (consumed by env.py)
--------------------------------------------
+grader.py – PhishGuard-Env SOC Triage Scoring Logic
+====================================================
+
+REWARD CONTRACT  →  OPEN INTERVAL (0.0, 1.0)
+---------------------------------------------
+All rewards are STRICTLY greater than 0 and STRICTLY less than 1.
+The endpoints 0 and 1 are NEVER returned.  This is a hard invariant
+enforced by the constant table below and by calculate_overall_score().
+
+Why open-interval?
+  • 1.0 saturates the leaderboard and implies a theoretically perfect agent.
+  • 0.0 is indistinguishable from a missing data-point in an RL pipeline.
+  • Every decision carries a non-zero gradient signal so training never dies.
+
+REWARD TABLE
+────────────────────────────────────────────────────────────────────────────
+  Constant                 Value   Outcome / Rationale
+  ─────────────────────────────────────────────────────────────────────────
+  R_PERFECT                0.95    Exact match — near-ideal; headroom for 1.0
+  R_MALWARE_QUARANTINE     0.75    MALWARE → QUARANTINE (textbook isolation)
+  R_PHISH_BEC_QUARANTINE   0.60    PHISH/BEC → QUARANTINE (domain still live)
+  R_SPAM_BLOCK             0.40    SPAM → BLOCK_DOMAIN (over-escalation)
+  R_SPAM_QUARANTINE        0.35    SPAM → QUARANTINE (lighter over-escalation)
+  R_WRONG_PROCEDURE        0.10    Wrong action, no direct security/ops harm
+  R_DISRUPTION             0.05    SAFE email blocked — operational cost
+  R_BREACH                 0.02    Threat allowed into inbox — catastrophic
+
+HEALTH-DRAIN THRESHOLD
+────────────────────────────────────────────────────────────────────────────
   HEALTH_DRAIN_THRESHOLD = 0.15
-  reward < 0.15  →  env.py deducts one life from the agent.
+  reward < 0.15  →  agent loses one life.
+
+PASS_THRESHOLD
+────────────────────────────────────────────────────────────────────────────
+  PASS_THRESHOLD = 0.50
+
+LEVEL CONTEXT (from env.py)
+────────────────────────────────────────────────────────────────────────────
+  easy   → lv1 (SPAM),    lv2 (PHISH),   lv3 (SAFE)
+  medium → lv4 (MALWARE), lv5 (SAFE),    lv6 (BEC),   lv7 (PHISH)
+  hard   → lv8 (MALWARE), lv9 (PHISH),   lv10 (BEC)
+
+VALID AGENT ACTIONS
+────────────────────────────────────────────────────────────────────────────
+  MARK_SAFE      – deliver to inbox
+  MOVE_TO_SPAM   – bulk / unsolicited mail
+  QUARANTINE     – hold for analyst review
+  BLOCK_DOMAIN   – perimeter block
+
+BUG FIX (v1.0.2 → v1.0.3)
+────────────────────────────────────────────────────────────────────────────
+  SPAM added to _THREAT_TYPES so MARK_SAFE on any threat drains health.
 """
 
-from typing import Tuple
-
-# ── Reward constants ─────��─────────────────────────────────────────────────────
-R_PERFECT               = 0.95
-R_MALWARE_QUARANTINE    = 0.75
-R_PHISH_BEC_QUARANTINE  = 0.60
-R_SPAM_BLOCK            = 0.40
-R_SPAM_QUARANTINE       = 0.35
-R_WRONG_PROCEDURE       = 0.10
-R_DISRUPTION            = 0.05
-R_BREACH                = 0.02
-
-# R_PARTIAL: alias for R_MALWARE_QUARANTINE.
-# test_grader.py imports this name and asserts R_PARTIAL == R_MALWARE_QUARANTINE.
-# Without this line, test_grader.py crashes with ImportError on line 21.
-R_PARTIAL               = R_MALWARE_QUARANTINE   # 0.75
-
-PASS_THRESHOLD          = 0.70   # inference.py imports this — a score >= this = success
-SUCCESS_THRESHOLD       = 0.70   # same value, alternative name
-
-HEALTH_DRAIN_THRESHOLD  = 0.15   # env.py: reward < this → lose one life
-
-
-# ── Task Registry ──────────────────────────────────────────────────────────────
-TASK_REGISTRY: dict[str, dict] = {
-    "task_spam": {
-        "email_types": {"SPAM"},
-        "description": (
-            "Identify unsolicited bulk/commercial mail and route it to spam "
-            "without over-escalating to a full domain block."
-        ),
-        "threat": True,
-    },
-    "task_phishing": {
-        "email_types": {"PHISH"},
-        "description": (
-            "Detect credential-harvesting and typosquat phishing attacks; "
-            "block the offending domain at the perimeter."
-        ),
-        "threat": True,
-    },
-    "task_safe": {
-        "email_types": {"SAFE"},
-        "description": (
-            "Recognise legitimate internal and partner emails and pass them "
-            "through without causing business disruption."
-        ),
-        "threat": False,
-    },
-    "task_malware": {
-        "email_types": {"MALWARE"},
-        "description": (
-            "Identify emails delivering executable payloads or macro-enabled "
-            "attachments and quarantine them for analysis."
-        ),
-        "threat": True,
-    },
-    "task_bec": {
-        "email_types": {"BEC"},
-        "description": (
-            "Detect business email compromise and supply-chain attacks, "
-            "including cases where SPF/DMARC authentication passes."
-        ),
-        "threat": True,
-    },
-}
-
-# Reverse lookup: raw email_type string → task_id
-_TYPE_TO_TASK_ID: dict[str, str] = {
-    etype: task_id
-    for task_id, meta in TASK_REGISTRY.items()
-    for etype in meta["email_types"]
-}
-
-
-def resolve_task_id(email_type: str) -> str:
-    """Convert raw email type (e.g. 'PHISH') to task_id (e.g. 'task_phishing')."""
-    key = email_type.strip().upper()
-    if key not in _TYPE_TO_TASK_ID:
-        raise ValueError(
-            f"Unknown email_type '{email_type}'. "
-            f"Valid types: {sorted(_TYPE_TO_TASK_ID)}"
-        )
-    return _TYPE_TO_TASK_ID[key]
+from __future__ import annotations
 
+import logging
+from typing import Tuple
 
-# ── Core grading function ──────────────────────────────────────────────────────
+logger = logging.getLogger(__name__)
+
+__all__ = [
+    "R_PERFECT",
+    "R_MALWARE_QUARANTINE",
+    "R_PHISH_BEC_QUARANTINE",
+    "R_SPAM_BLOCK",
+    "R_SPAM_QUARANTINE",
+    "R_WRONG_PROCEDURE",
+    "R_DISRUPTION",
+    "R_BREACH",
+    "R_PARTIAL",
+    "PASS_THRESHOLD",
+    "HEALTH_DRAIN_THRESHOLD",
+    "grade_action",
+    "grade_easy",
+    "grade_medium",
+    "grade_hard",
+    "grade_performance",
+    "calculate_overall_score",
+    "GRADERS",
+    "SCENARIO_LOADERS",
+]
+
+
+# ══════════════════════════════════════════════════════════════════════════════
+# REWARD CONSTANTS
+# ══════════════════════════════════════════════════════════════════════════════
+
+R_PERFECT              = 0.95
+R_MALWARE_QUARANTINE   = 0.75
+R_PHISH_BEC_QUARANTINE = 0.60
+R_SPAM_BLOCK           = 0.40
+R_SPAM_QUARANTINE      = 0.35
+R_WRONG_PROCEDURE      = 0.10
+R_DISRUPTION           = 0.05
+R_BREACH               = 0.02
+
+R_PARTIAL              = R_MALWARE_QUARANTINE
+PASS_THRESHOLD         = 0.50
+HEALTH_DRAIN_THRESHOLD = 0.15
+
+_THREAT_TYPES  = frozenset({"PHISH", "BEC", "MALWARE", "SPAM"})
+_BLOCKED_MOVES = frozenset({"BLOCK_DOMAIN", "QUARANTINE", "MOVE_TO_SPAM"})
+_VALID_ACTIONS = frozenset({"MARK_SAFE", "MOVE_TO_SPAM", "QUARANTINE", "BLOCK_DOMAIN"})
+
+
+# ══════════════════════════════════════════════════════════════════════════════
+# GRADE_ACTION
+# ══════════════════════════════════════════════════════════════════════════════
 
 def grade_action(
     agent_output: str,
@@ -117,108 +115,289 @@ def grade_action(
     email_type: str,
 ) -> Tuple[float, str]:
     """
-    Grade one SOC triage decision.
-    Returns (reward, message) where reward is strictly in (0.0, 1.0).
+    Grade one SOC triage decision and return a reward in (0.0, 1.0).
+
+    Parameters
+    ----------
+    agent_output    : Raw action string from the LLM agent (case-insensitive).
+    expected_output : Ground-truth correct action for this scenario.
+    email_type      : Threat category — PHISH | BEC | MALWARE | SPAM | SAFE.
+
+    Returns
+    -------
+    (reward, message)
+        reward  : float strictly in (0.0, 1.0)
+        message : short verdict string for logs and agent feedback.
     """
     agent_action    = agent_output.strip().upper()
     expected_action = expected_output.strip().upper()
     etype           = email_type.strip().upper()
 
-    # Validate action is one of the four known values
-    valid_actions = {"MARK_SAFE", "MOVE_TO_SPAM", "QUARANTINE", "BLOCK_DOMAIN"}
-    if agent_action not in valid_actions:
-        return R_WRONG_PROCEDURE, f"INVALID_ACTION: '{agent_action}' is not a recognised triage action"
-
-    task_id   = resolve_task_id(etype)
-    is_threat = TASK_REGISTRY[task_id]["threat"]
+    # Step 1: Reject unrecognised tokens
+    if agent_action not in _VALID_ACTIONS:
+        return (
+            R_WRONG_PROCEDURE,
+            f"INVALID_ACTION: '{agent_action}' is not a recognised triage action — "
+            f"must be one of: {', '.join(sorted(_VALID_ACTIONS))}",
+        )
 
-    # ── 1. Perfect match ───────────────────────────────────────────────────────
+    # Step 2: Perfect match
     if agent_action == expected_action:
         return R_PERFECT, "PERFECT_TRIAGE: Correct action taken"
 
-    # ── 2. Security Breach ─────────────────────────────────────────────────────
-    if is_threat and agent_action == "MARK_SAFE":
-        return R_BREACH, "SECURITY_BREACH: Threat allowed into the network"
+    # Step 3: Security Breach
+    if etype in _THREAT_TYPES and agent_action == "MARK_SAFE":
+        return (
+            R_BREACH,
+            f"SECURITY_BREACH: {etype} threat delivered to inbox unimpeded — "
+            "catastrophic failure; source remains active",
+        )
+
+    # Step 4: Business Disruption
+    if etype == "SAFE" and agent_action in _BLOCKED_MOVES:
+        return (
+            R_DISRUPTION,
+            "BUSINESS_DISRUPTION: Legitimate communication was incorrectly blocked — "
+            "operational impact; sender trust degraded",
+        )
 
-    # ── 3. Business Disruption ─────────────────────────────────────────────────
-    if not is_threat and agent_action in {"BLOCK_DOMAIN", "QUARANTINE", "MOVE_TO_SPAM"}:
-        return R_DISRUPTION, "BUSINESS_DISRUPTION: Legitimate communication blocked"
+    # Step 5: Partial credit
 
-    # ── 4. Partial credit ──────────────────────────────────────────────────────
-    if task_id == "task_malware" and agent_action == "QUARANTINE":
-        return R_MALWARE_QUARANTINE, "CAUTIOUS: Malware isolated — strong containment"
+    # MALWARE → QUARANTINE
+    if etype == "MALWARE" and agent_action == "QUARANTINE":
+        return (
+            R_MALWARE_QUARANTINE,
+            "CAUTIOUS: Malware isolated via QUARANTINE — strong containment; "
+            "no further propagation risk detected",
+        )
 
-    if task_id in {"task_phishing", "task_bec"} and agent_action == "QUARANTINE":
-        return R_PHISH_BEC_QUARANTINE, (
-            "UNDER_RESPONSE: Threat quarantined but domain still active — "
-            "consider BLOCK_DOMAIN"
+    # PHISH/BEC → QUARANTINE
+    if etype in {"PHISH", "BEC"} and agent_action == "QUARANTINE":
+        return (
+            R_PHISH_BEC_QUARANTINE,
+            f"UNDER_RESPONSE: {etype} quarantined but source domain still active — "
+            "prefer BLOCK_DOMAIN to prevent further phishing delivery attempts",
         )
 
-    if task_id == "task_spam" and agent_action == "BLOCK_DOMAIN":
-        return R_SPAM_BLOCK, (
+    # SPAM → BLOCK_DOMAIN
+    if etype == "SPAM" and agent_action == "BLOCK_DOMAIN":
+        return (
+            R_SPAM_BLOCK,
             "OVER_ESCALATION: BLOCK_DOMAIN is disproportionate for SPAM — "
-            "prefer MOVE_TO_SPAM"
+            "prefer MOVE_TO_SPAM to preserve block-list resources",
         )
 
-    if task_id == "task_spam" and agent_action == "QUARANTINE":
-        return R_SPAM_QUARANTINE, (
-            "OVER_ESCALATION: QUARANTINE wastes analyst capacity for SPAM — "
-            "prefer MOVE_TO_SPAM"
+    # SPAM → QUARANTINE
+    if etype == "SPAM" and agent_action == "QUARANTINE":
+        return (
+            R_SPAM_QUARANTINE,
+            "OVER_ESCALATION: QUARANTINE wastes analyst capacity on SPAM — "
+            "prefer MOVE_TO_SPAM for bulk unsolicited mail",
         )
 
-    # ── 5. General wrong procedure ─────────────────────────────────────────────
-    return R_WRONG_PROCEDURE, "INCORRECT_PROCEDURE: Decision does not match security policy"
+    # Step 6: Catch-all wrong procedure
+    return (
+        R_WRONG_PROCEDURE,
+        f"INCORRECT_PROCEDURE: '{agent_action}' does not match security policy "
+        f"for {etype} email (expected: {expected_action}) — "
+        "review triage guidelines",
+    )
 
 
-# ── Episode-level score aggregation ───────────────────────────────────────────
+# ══════════════════════════════════════════════════════════════════════════════
+# CALCULATE_OVERALL_SCORE
+# ══════════════════════════════════════════════════════════════════════════════
 
 def calculate_overall_score(task_scores: list) -> float:
-    """Compute final benchmark score from per-step rewards. Returns float in (0,1)."""
+    """
+    Compute the final benchmark score from a list of per-step rewards.
+    Result is clamped to [R_BREACH, R_PERFECT].
+    Empty list returns R_BREACH.
+    """
     if not task_scores:
         return R_BREACH
+
     raw_avg = sum(task_scores) / len(task_scores)
     clamped = max(R_BREACH, min(R_PERFECT, raw_avg))
     return round(clamped, 4)
 
 
-def calculate_per_task_scores(
-    task_score_map: dict[str, list[float]],
-) -> dict[str, float]:
-    """Compute per-task-type average scores. Returns {task_id: score}."""
-    return {
-        task_id: calculate_overall_score(scores)
-        for task_id, scores in task_score_map.items()
-        if scores
-    }
+# ══════════════════════════════════════════════════════════════════════════════
+# SCORE SAFETY HELPERS
+# ══════════════════════════════════════════════════════════════════════════════
 
+def _safe_score(raw: float) -> float:
+    """
+    Map any float to the open interval (R_BREACH, R_PERFECT).
 
-# ── Named graders — one per difficulty level ──────────────────────────────────
-# Referenced by openenv.yaml:
-#   task id "easy"   → grader: "grader.grade_easy"
-#   task id "medium" → grader: "grader.grade_medium"
-#   task id "hard"   → grader: "grader.grade_hard"
-#
-# The validator does: import grader; callable(grader.grade_easy) → True
-# Without these, it counts 0 graded tasks → "Not enough tasks with graders"
+    Mirrors Focus-AI's safe_score() pattern:
+        safe_score(raw) = LOWER + (UPPER - LOWER) * clamp(raw, 0, 1)
+    where LOWER = R_BREACH (0.02), UPPER = R_PERFECT (0.95).
 
-def grade_easy(task_scores: list) -> float:
-    """Grader for easy-level scenarios (lv1–lv3: SPAM, PHISH, SAFE). Returns float in (0, 1)."""
-    return calculate_overall_score(task_scores)
+    Guarantees:
+        raw = 0.0  ->  0.02   (> 0, never equals 0)
+        raw = 1.0  ->  0.95   (< 1, never equals 1)
+    """
+    raw = float(raw)
+    if raw < 0.0:
+        raw = 0.0
+    elif raw > 1.0:
+        raw = 1.0
+    result = R_BREACH + (R_PERFECT - R_BREACH) * raw
+    result = round(result, 6)
+    assert 0.0 < result < 1.0, (
+        f"_safe_score VIOLATION: raw={raw!r} produced result={result!r} "
+        f"which is not strictly inside (0, 1)"
+    )
+    return result
+
+
+def _safe_ratio(num: float, den: float) -> float:
+    """Safe division clamped to [0, 1]."""
+    if den <= 0:
+        return 0.0
+    return max(0.0, min(1.0, num / den))
+
+
+# ══════════════════════════════════════════════════════════════════════════════
+# OPENENV GRADERS
+# Called by the OpenEnv validator — one function per difficulty level.
+# Signature: grade_X(metrics: dict) -> float strictly in (0, 1)
+# ══════════════════════════════════════════════════════════════════════════════
+
+def grade_easy(metrics: dict) -> float:
+    """
+    Grader for easy tasks (lv1-lv3): SPAM, PHISH, SAFE.
+    Scoring: 60% correct action + 40% threat identification accuracy.
+    """
+    total   = max(1, metrics.get("total_tasks", metrics.get("total", 3)))
+    correct = metrics.get("correct_actions", metrics.get("is_correct", 0))
+    on_time = metrics.get("on_time", metrics.get("completed", correct))
 
+    if isinstance(correct, bool):
+        correct = int(correct)
+    if isinstance(on_time, bool):
+        on_time = int(on_time)
 
-def grade_medium(task_scores: list) -> float:
-    """Grader for medium-level scenarios (lv4–lv7: MALWARE, SAFE, BEC, PHISH). Returns float in (0, 1)."""
-    return calculate_overall_score(task_scores)
+    raw = (
+        0.60 * _safe_ratio(correct, total)
+        + 0.40 * _safe_ratio(on_time, total)
+    )
+    return _safe_score(raw)
 
 
-def grade_hard(task_scores: list) -> float:
-    """Grader for hard-level scenarios (lv8–lv10: MALWARE, PHISH, BEC). Returns float in (0, 1)."""
-    return calculate_overall_score(task_scores)
+def grade_medium(metrics: dict) -> float:
+    """
+    Grader for medium tasks (lv4-lv7): MALWARE, SAFE HR, BEC, PHISH.
+    Scoring: 40% correct + 35% on-time detection + 25% escalation quality.
+    """
+    total    = max(1, metrics.get("total_tasks", metrics.get("total", 4)))
+    correct  = metrics.get("correct_actions", metrics.get("is_correct", 0))
+    on_time  = metrics.get("on_time", metrics.get("completed", correct))
+    steps    = max(1, metrics.get("total_steps", metrics.get("steps", 4)))
+    good_esc = metrics.get("good_escalation", metrics.get("reward", correct))
+
+    if isinstance(correct,  bool): correct  = int(correct)
+    if isinstance(on_time,  bool): on_time  = int(on_time)
+    if isinstance(good_esc, bool): good_esc = int(good_esc)
 
+    raw = (
+        0.40 * _safe_ratio(correct,  total)
+        + 0.35 * _safe_ratio(on_time,  total)
+        + 0.25 * _safe_ratio(good_esc, steps)
+    )
+    return _safe_score(raw)
+
+
+def grade_hard(metrics: dict) -> float:
+    """
+    Grader for hard tasks (lv8-lv10): MALWARE macro, QR phishing, BEC domain.
+    Scoring: 35% correct + 30% threat accuracy + 20% escalation + 15% priority.
+    """
+    total    = max(1, metrics.get("total_tasks", metrics.get("total", 3)))
+    correct  = metrics.get("correct_actions", metrics.get("is_correct", 0))
+    on_time  = metrics.get("on_time", metrics.get("completed", correct))
+    steps    = max(1, metrics.get("total_steps", metrics.get("steps", 3)))
+    good_esc = metrics.get("good_escalation", metrics.get("reward", correct))
+    hi_pri   = metrics.get("high_priority_correct", correct)
+
+    if isinstance(correct,  bool): correct  = int(correct)
+    if isinstance(on_time,  bool): on_time  = int(on_time)
+    if isinstance(good_esc, bool): good_esc = int(good_esc)
+    if isinstance(hi_pri,   bool): hi_pri   = int(hi_pri)
+
+    raw = (
+        0.35 * _safe_ratio(correct,  total)
+        + 0.30 * _safe_ratio(on_time,  total)
+        + 0.20 * _safe_ratio(good_esc, steps)
+        + 0.15 * _safe_ratio(hi_pri,   max(1, correct))
+    )
+    return _safe_score(raw)
+
+
+def grade_performance(metrics: dict) -> float:
+    """
+    Aggregate grader used for cross-difficulty scoring.
+
+    Mirrors Focus-AI's grade_performance() pattern — provides a single
+    unified score across all difficulty levels for leaderboard ranking.
+
+    Scoring: 40% correct actions + 30% on-time + 20% escalation + 10% priority.
+    """
+    total    = max(1, metrics.get("total_tasks", metrics.get("total", 1)))
+    correct  = metrics.get("correct_actions", metrics.get("is_correct", 0))
+    on_time  = metrics.get("on_time", metrics.get("completed", correct))
+    steps    = max(1, metrics.get("total_steps", metrics.get("steps", 1)))
+    good_esc = metrics.get("good_escalation", metrics.get("reward", correct))
+
+    if isinstance(correct,  bool): correct  = int(correct)
+    if isinstance(on_time,  bool): on_time  = int(on_time)
+    if isinstance(good_esc, bool): good_esc = int(good_esc)
+
+    raw = (
+        0.40 * _safe_ratio(correct,  total)
+        + 0.30 * _safe_ratio(on_time,  total)
+        + 0.20 * _safe_ratio(good_esc, steps)
+        + 0.10 * _safe_ratio(correct,  steps)
+    )
+    return _safe_score(raw)
+
+
+# ══════════════════════════════════════════════════════════════════════════════
+# GRADERS DICT  (mirrors Focus-AI's GRADERS pattern)
+# Maps difficulty level → grader function for easy programmatic lookup.
+# ══════════════════════════════════════════════════════════════════════════════
 
-# Registry map — consumed by env.py and server/app.py
 GRADERS = {
     "easy":   grade_easy,
     "medium": grade_medium,
     "hard":   grade_hard,
+}
+
+
+# ══════════════════════════════════════════════════════════════════════════════
+# SCENARIO LOADERS  (mirrors Focus-AI's TASK_LOADERS pattern)
+# Maps difficulty level → callable that returns the scenario list for that level.
+# Used by env.py to load scenarios without hard-coding level names.
+# ══════════════════════════════════════════════════════════════════════════════
+
+def _get_easy_scenarios() -> list:
+    """Return scenario IDs for easy difficulty."""
+    return ["lv1", "lv2", "lv3"]
+
+
+def _get_medium_scenarios() -> list:
+    """Return scenario IDs for medium difficulty."""
+    return ["lv4", "lv5", "lv6", "lv7"]
+
+
+def _get_hard_scenarios() -> list:
+    """Return scenario IDs for hard difficulty."""
+    return ["lv8", "lv9", "lv10"]
+
+
+SCENARIO_LOADERS = {
+    "easy":   _get_easy_scenarios,
+    "medium": _get_medium_scenarios,
+    "hard":   _get_hard_scenarios,
 }