Upload 18 files

.dockerignore

@@ -0,0 +1,34 @@
+# Git metadata — no need inside the image
+.git/
+.gitignore
+.gitattributes
+
+# Python cache — rebuilt inside the image
+__pycache__/
+*.py[cod]
+*.pyo
+
+# Package manager lock files — not needed at runtime
+uv.lock
+poetry.lock
+
+# IDE / OS noise
+.vscode/
+.idea/
+.DS_Store
+
+# Test files — not needed in production image
+test_*.py
+tests/
+
+# Inference agent — client-side only, not needed in server image
+inference.py
+
+# Environment variable files — may contain secrets
+.env*
+
+# Inference run outputs
+results_*.json
+
+# Docs
+README.md

.env.example

@@ -0,0 +1,14 @@
+# PhishGuard-Env – environment variable reference
+# Copy this file to .env and fill in your values.
+# Never commit the actual .env file (it's in .gitignore).
+
+# ── LLM API credentials (one of these is required for inference.py) ───────────
+HF_TOKEN=hf_your_token_here
+# OPENAI_API_KEY=sk-your_key_here   # alternative to HF_TOKEN
+
+# ── LLM / API settings ────────────────────────────────────────────────────────
+API_BASE_URL=https://router.huggingface.co/v1
+MODEL_NAME=Qwen/Qwen2.5-72B-Instruct
+
+# ── Environment server URL (used by inference.py to reach env.py) ─────────────
+ENV_BASE_URL=http://localhost:7860

.gitattributes

@@ -1,35 +1,35 @@
-*.7z filter=lfs diff=lfs merge=lfs -text
-*.arrow filter=lfs diff=lfs merge=lfs -text
-*.bin filter=lfs diff=lfs merge=lfs -text
-*.bz2 filter=lfs diff=lfs merge=lfs -text
-*.ckpt filter=lfs diff=lfs merge=lfs -text
-*.ftz filter=lfs diff=lfs merge=lfs -text
-*.gz filter=lfs diff=lfs merge=lfs -text
-*.h5 filter=lfs diff=lfs merge=lfs -text
-*.joblib filter=lfs diff=lfs merge=lfs -text
-*.lfs.* filter=lfs diff=lfs merge=lfs -text
-*.mlmodel filter=lfs diff=lfs merge=lfs -text
-*.model filter=lfs diff=lfs merge=lfs -text
-*.msgpack filter=lfs diff=lfs merge=lfs -text
-*.npy filter=lfs diff=lfs merge=lfs -text
-*.npz filter=lfs diff=lfs merge=lfs -text
-*.onnx filter=lfs diff=lfs merge=lfs -text
-*.ot filter=lfs diff=lfs merge=lfs -text
-*.parquet filter=lfs diff=lfs merge=lfs -text
-*.pb filter=lfs diff=lfs merge=lfs -text
-*.pickle filter=lfs diff=lfs merge=lfs -text
-*.pkl filter=lfs diff=lfs merge=lfs -text
-*.pt filter=lfs diff=lfs merge=lfs -text
-*.pth filter=lfs diff=lfs merge=lfs -text
-*.rar filter=lfs diff=lfs merge=lfs -text
-*.safetensors filter=lfs diff=lfs merge=lfs -text
-saved_model/**/* filter=lfs diff=lfs merge=lfs -text
-*.tar.* filter=lfs diff=lfs merge=lfs -text
-*.tar filter=lfs diff=lfs merge=lfs -text
-*.tflite filter=lfs diff=lfs merge=lfs -text
-*.tgz filter=lfs diff=lfs merge=lfs -text
-*.wasm filter=lfs diff=lfs merge=lfs -text
-*.xz filter=lfs diff=lfs merge=lfs -text
-*.zip filter=lfs diff=lfs merge=lfs -text
-*.zst filter=lfs diff=lfs merge=lfs -text
-*tfevents* filter=lfs diff=lfs merge=lfs -text
+*.7z filter=lfs diff=lfs merge=lfs -text
+*.arrow filter=lfs diff=lfs merge=lfs -text
+*.bin filter=lfs diff=lfs merge=lfs -text
+*.bz2 filter=lfs diff=lfs merge=lfs -text
+*.ckpt filter=lfs diff=lfs merge=lfs -text
+*.ftz filter=lfs diff=lfs merge=lfs -text
+*.gz filter=lfs diff=lfs merge=lfs -text
+*.h5 filter=lfs diff=lfs merge=lfs -text
+*.joblib filter=lfs diff=lfs merge=lfs -text
+*.lfs.* filter=lfs diff=lfs merge=lfs -text
+*.mlmodel filter=lfs diff=lfs merge=lfs -text
+*.model filter=lfs diff=lfs merge=lfs -text
+*.msgpack filter=lfs diff=lfs merge=lfs -text
+*.npy filter=lfs diff=lfs merge=lfs -text
+*.npz filter=lfs diff=lfs merge=lfs -text
+*.onnx filter=lfs diff=lfs merge=lfs -text
+*.ot filter=lfs diff=lfs merge=lfs -text
+*.parquet filter=lfs diff=lfs merge=lfs -text
+*.pb filter=lfs diff=lfs merge=lfs -text
+*.pickle filter=lfs diff=lfs merge=lfs -text
+*.pkl filter=lfs diff=lfs merge=lfs -text
+*.pt filter=lfs diff=lfs merge=lfs -text
+*.pth filter=lfs diff=lfs merge=lfs -text
+*.rar filter=lfs diff=lfs merge=lfs -text
+*.safetensors filter=lfs diff=lfs merge=lfs -text
+saved_model/**/* filter=lfs diff=lfs merge=lfs -text
+*.tar.* filter=lfs diff=lfs merge=lfs -text
+*.tar filter=lfs diff=lfs merge=lfs -text
+*.tflite filter=lfs diff=lfs merge=lfs -text
+*.tgz filter=lfs diff=lfs merge=lfs -text
+*.wasm filter=lfs diff=lfs merge=lfs -text
+*.xz filter=lfs diff=lfs merge=lfs -text
+*.zip filter=lfs diff=lfs merge=lfs -text
+*.zst filter=lfs diff=lfs merge=lfs -text
+*tfevents* filter=lfs diff=lfs merge=lfs -text

.gitignore

@@ -0,0 +1,35 @@
+# Python
+__pycache__/
+*.py[cod]
+*.pyo
+*.pyd
+.Python
+*.egg-info/
+dist/
+build/
+.eggs/
+
+# Virtual environments
+.venv/
+venv/
+env/
+.env
+
+# Package managers
+uv.lock
+poetry.lock
+
+# IDE / editor
+.vscode/
+.idea/
+*.swp
+*.swo
+.DS_Store
+
+# Test / coverage
+.pytest_cache/
+.coverage
+htmlcov/
+
+# Inference run outputs
+results_*.json

Dockerfile

@@ -0,0 +1,28 @@
+# BUG FIX: pin full patch version for reproducible image builds.
+# `python:3.10-slim` floats to whatever the latest 3.10.x is at build time.
+FROM python:3.10.14-slim
+
+WORKDIR /app
+
+# Install dependencies before copying source so Docker layer cache is reused
+# on code-only changes.
+COPY requirements.txt .
+RUN pip install --no-cache-dir --upgrade pip && \
+    pip install --no-cache-dir -r requirements.txt
+
+COPY . .
+
+# BUG FIX: run as a non-root user — running as root in a container is a
+# security risk and violates the principle of least privilege.
+RUN adduser --disabled-password --gecos "" appuser
+USER appuser
+
+EXPOSE 7860
+
+# BUG FIX: declare a HEALTHCHECK so HF Spaces / orchestrators can detect
+# if the server has crashed and restart the container automatically.
+HEALTHCHECK --interval=30s --timeout=5s --start-period=10s --retries=3 \
+  CMD python -c "import urllib.request; urllib.request.urlopen('http://localhost:7860/health')" \
+  || exit 1
+
+CMD ["uvicorn", "server.app:app", "--host", "0.0.0.0", "--port", "7860"]

README.md

@@ -1,12 +1,106 @@
----
-title: Phishing Env 2
-emoji: 📈
-colorFrom: red
-colorTo: yellow
-sdk: docker
-pinned: false
-license: mit
-short_description: Side project
----
-
-Check out the configuration reference at https://huggingface.co/docs/hub/spaces-config-reference
+# PhishGuard-Env
+
+An OpenEnv-compliant SOC analyst simulation environment for benchmarking LLM-based email triage agents.
+
+## Overview
+
+PhishGuard-Env presents an agent with phishing, malware, BEC, spam, and safe emails. The agent must classify each one using a four-action triage system. Rewards are graded continuously in `(0.0, 1.0)` — never 0, never 1 — and a health system penalises critical mistakes.
+
+## Difficulty Levels
+
+| Level  | Scenarios       | Tasks |
+|--------|-----------------|-------|
+| easy   | lv1–lv3         | 3     |
+| medium | lv4–lv7         | 4     |
+| hard   | lv8–lv10        | 3     |
+
+## Action Space
+
+| Action        | When to use                                      |
+|---------------|--------------------------------------------------|
+| `MARK_SAFE`   | Confirmed legitimate email — deliver to inbox    |
+| `MOVE_TO_SPAM`| Bulk / unsolicited mail, no active threat        |
+| `QUARANTINE`  | Suspicious but unconfirmed — hold for review     |
+| `BLOCK_DOMAIN`| Confirmed phishing / BEC / malware source        |
+
+## Quickstart
+
+### 1. Clone and install
+
+```bash
+git clone https://github.com/your-username/phishguard-env
+cd phishguard-env
+pip install -r requirements.txt
+```
+
+### 2. Configure credentials
+
+```bash
+cp .env.example .env
+# Edit .env and set HF_TOKEN or OPENAI_API_KEY
+```
+
+### 3. Start the environment server
+
+```bash
+python env.py
+# Server starts on http://localhost:7860
+```
+
+### 4. Run the inference agent
+
+```bash
+# All three levels
+python inference.py
+
+# Single level
+python inference.py --level hard
+
+# Custom output path
+python inference.py --output my_run.json
+```
+
+### 5. Run tests
+
+```bash
+pytest test_grader.py -v
+```
+
+## Docker
+
+```bash
+docker build -t phishguard-env .
+docker run -p 7860:7860 -e HF_TOKEN=hf_... phishguard-env
+```
+
+## API Reference
+
+### `POST /reset`
+Start a new episode at the chosen difficulty level. The request body is **optional** — if omitted, defaults to `"easy"`.
+
+```json
+{ "level": "easy" }
+```
+
+### `POST /step`
+Submit one triage action.
+
+```json
+{ "action": "BLOCK_DOMAIN", "reasoning": "Domain registered 3 days ago with SPF fail." }
+```
+
+### `GET /state`
+Read-only snapshot of current environment state.
+
+### `GET /health`
+Liveness probe for HF Spaces / load-balancers.
+
+## Environment Variables
+
+| Variable          | Default                              | Description                    |
+|-------------------|--------------------------------------|--------------------------------|
+| `HF_TOKEN`        | —                                    | Hugging Face API key           |
+| `OPENAI_API_KEY`  | —                                    | OpenAI-compatible API key      |
+| `API_BASE_URL`    | `https://router.huggingface.co/v1`   | LLM API base URL               |
+| `MODEL_NAME`      | `Qwen/Qwen2.5-72B-Instruct`          | Model to use for inference     |
+| `ENV_BASE_URL`    | `http://localhost:7860`              | Environment server URL         |

env.py

@@ -0,0 +1,739 @@
+"""
+env.py – PhishGuard-Env  |  FastAPI Environment Server
+=======================================================
+
+ARCHITECTURE ROLE
+-----------------
+This file IS the environment. It runs as a persistent FastAPI server on
+Hugging Face Spaces (port 7860). The inference agent (inference.py) is a
+separate process that interacts with it exclusively through HTTP — it NEVER
+imports this module directly.
+
+Endpoints
+---------
+  POST /reset   → reset for a chosen difficulty level; returns first email observation
+  POST /step    → submit one triage action; returns (obs, reward, done, info)
+  GET  /state   → read-only snapshot of health, score, task index
+  GET  /health  → liveness probe for HF Spaces / load-balancers
+
+LEVEL DESIGN
+------------
+  easy   → lv1, lv2, lv3         (3 tasks)
+  medium → lv4, lv5, lv6, lv7   (4 tasks)
+  hard   → lv8, lv9, lv10       (3 tasks)
+
+State variables
+---------------
+  current_task_idx  : int   – index into the active scenario list
+  health            : int   – lives remaining (starts at 3)
+  score             : float – cumulative reward for this episode
+  task_scores       : list  – per-step reward history
+
+Reward contract
+---------------
+All rewards are sourced from grader.py and strictly in the open interval
+(0.0, 1.0). No endpoint ever returns 0 or 1.
+
+Health drain
+------------
+HEALTH_DRAIN_THRESHOLD = 0.15 (imported from grader).
+Any step reward below this threshold costs one life.
+  Security Breach (0.02)       → –1 life
+  Business Disruption (0.05)   → –1 life
+  Wrong Procedure (0.10)       → –1 life
+  Cautious / partial (≥ 0.35)  → no life loss
+
+Thread safety
+-------------
+_env is a singleton shared across all FastAPI requests. The asyncio.Lock
+`_env_lock` serialises access to _env so concurrent /step or /reset calls
+cannot race on current_task_idx, health, score, or task_scores.
+
+OpenEnv validator compliance
+-----------------------------
+Every normal /step response includes:
+  task_id    : str  – the scenario id (e.g. "lv3") so the validator can
+                      correlate decisions to tasks.
+  is_correct : bool – True when reward >= R_PERFECT, so the validator
+                      can count "tasks with graders" (≥ 3 required).
+
+BUG FIXES (v1.0.2 → v1.0.3)
+-----------------------------
+  • Thread-safety: asyncio.Lock added around all _env mutations.
+  • total_tasks in /reset now correctly reports len(LEVEL_MAP[level])
+    (the count for the chosen level only) instead of len(_env.scenarios)
+    which could include stale data from a previous reset.
+"""
+
+from __future__ import annotations
+
+import asyncio
+import logging
+import random
+from contextlib import asynccontextmanager
+from typing import List, Optional
+
+from fastapi import FastAPI, HTTPException
+from fastapi.middleware.cors import CORSMiddleware
+
+from models import PhishAction, ResetRequest, StepResponse, ResetResponse
+
+# ── Logging ───────────────────────────────────────────────────────────────────
+logging.basicConfig(
+    level=logging.INFO,
+    format="%(asctime)s | %(levelname)s | %(message)s",
+    datefmt="%H:%M:%S",
+)
+log = logging.getLogger("phishguard.env")
+
+# ── OpenEnv base-class (graceful degradation) ─────────────────────────────────
+try:
+    from openenv import OpenEnv
+except ImportError:
+    try:
+        from openenv.core import OpenEnv  # type: ignore
+    except ImportError:
+        OpenEnv = object  # Runs as a plain Python class if openenv is absent
+
+# ── Grader imports ────────────────────────────────────────────────────────────
+from grader import (
+    R_BREACH,
+    R_PERFECT,
+    HEALTH_DRAIN_THRESHOLD,
+    calculate_overall_score,
+    grade_action,
+    grade_performance,
+    GRADERS,
+    SCENARIO_LOADERS,
+)
+
+VERSION = "1.0.3"
+
+# ═════════════════════════════════════════════════════════════════════════════
+# SCENARIO DEFINITIONS  (lv1 → lv10)
+# ═════════════════════════════════════════════════════════════════════════════
+
+SCENARIOS: List[dict] = [
+    # ── EASY (lv1–lv3) ───────────────────────────────────────────────────────
+    {
+        "id": "lv1",
+        "level": "easy",
+        "type": "SPAM",
+        "data": {
+            "sender": "win@lotto.net",
+            "subject": "Claim $1M Prize – Congratulations!",
+            "body": (
+                "You have been selected. "
+                "Click now to claim your prize before it expires."
+            ),
+            "links": ["http://bit.ly/123abc"],
+            "has_attachments": False,
+            "spf_record": "fail",
+            "dmarc_record": "none",
+            "urgency_level": "high",
+            "confidence_hint": "SIEM: no prior communication history with this domain",
+        },
+        "correct": "MOVE_TO_SPAM",
+    },
+    {
+        "id": "lv2",
+        "level": "easy",
+        "type": "PHISH",
+        "data": {
+            "sender": "support@googIe.com",
+            "subject": "Urgent Security Alert – Verify Your Account",
+            "body": (
+                "We detected suspicious activity on your account. "
+                "Verify your identity immediately."
+            ),
+            "links": ["http://googIe-verify.com/login"],
+            "has_attachments": False,
+            "spf_record": "softfail",
+            "dmarc_record": "fail",
+            "urgency_level": "critical",
+            "confidence_hint": "Gateway: domain registered 3 days ago, 0 prior sends",
+        },
+        "correct": "BLOCK_DOMAIN",
+    },
+    {
+        "id": "lv3",
+        "level": "easy",
+        "type": "SAFE",
+        "data": {
+            "sender": "boss@company.com",
+            "subject": "Team Meeting Tomorrow at 9 AM",
+            "body": (
+                "Hi team, please be ready for our weekly sync at 9 AM "
+                "in the main boardroom."
+            ),
+            "links": [],
+            "has_attachments": False,
+            "spf_record": "pass",
+            "dmarc_record": "pass",
+            "urgency_level": "low",
+            "confidence_hint": "SIEM: sender in address book for 2+ years, 200+ prior emails",
+        },
+        "correct": "MARK_SAFE",
+    },
+
+    # ── MEDIUM (lv4–lv7) ─────────────────────────────────────────────────────
+    {
+        "id": "lv4",
+        "level": "medium",
+        "type": "MALWARE",
+        "data": {
+            "sender": "it@internal-help.com",
+            "subject": "Critical Security Patch – Action Required Immediately",
+            "body": (
+                "A critical vulnerability has been discovered. "
+                "Run fix.exe to patch your system now."
+            ),
+            "links": ["http://internal-help.com/fix.exe"],
+            "has_attachments": True,
+            "spf_record": "fail",
+            "dmarc_record": "none",
+            "urgency_level": "critical",
+            "confidence_hint": "AV: attachment flagged as suspicious PE binary (confidence 87%)",
+        },
+        "correct": "QUARANTINE",
+    },
+    {
+        "id": "lv5",
+        "level": "medium",
+        "type": "SAFE",
+        "data": {
+            "sender": "hr@yourcompany.com",
+            "subject": "Updated Holiday Schedule – Please Review",
+            "body": (
+                "Hi all, the updated holiday list is now live on the HR portal. "
+                "No action needed."
+            ),
+            "links": ["https://portal.yourcompany.com/holidays"],
+            "has_attachments": False,
+            "spf_record": "pass",
+            "dmarc_record": "pass",
+            "urgency_level": "low",
+            "confidence_hint": (
+                "SIEM: bulk-sent to all-staff distribution list, "
+                "normal recurring pattern"
+            ),
+        },
+        "correct": "MARK_SAFE",
+    },
+    {
+        "id": "lv6",
+        "level": "medium",
+        "type": "BEC",
+        "data": {
+            "sender": "ceo@executive-mail.com",
+            "subject": "URGENT – Wire Transfer Required Today",
+            "body": (
+                "I'm in a meeting and cannot be reached by phone. "
+                "Wire $5,000 to the account below immediately."
+            ),
+            "links": [],
+            "has_attachments": False,
+            "spf_record": "pass",
+            "dmarc_record": "none",
+            "urgency_level": "critical",
+            "confidence_hint": "SIEM: first-ever email from this domain to any company inbox",
+        },
+        "correct": "QUARANTINE",
+    },
+    {
+        "id": "lv7",
+        "level": "medium",
+        "type": "PHISH",
+        "data": {
+            "sender": "notify@docs-share.net",
+            "subject": "You Have a New Shared Document",
+            "body": (
+                "Your colleague shared a file with you. "
+                "Click the link below to view it in your browser."
+            ),
+            "links": ["https://docs-verify.net/s/1x9az"],
+            "has_attachments": False,
+            "spf_record": "softfail",
+            "dmarc_record": "fail",
+            "urgency_level": "medium",
+            "confidence_hint": (
+                "Gateway: destination link redirects through 2 URL shorteners "
+                "before landing"
+            ),
+        },
+        "correct": "BLOCK_DOMAIN",
+    },
+
+    # ── HARD (lv8–lv10) ──────────────────────────────────────────────────────
+    {
+        "id": "lv8",
+        "level": "hard",
+        "type": "MALWARE",
+        "data": {
+            "sender": "payroll@hr-dept.net",
+            "subject": "Your Bank Details Form – Action Required",
+            "body": (
+                "Please review and complete the attached payroll update form "
+                "before Friday."
+            ),
+            "links": [],
+            "has_attachments": True,
+            "spf_record": "fail",
+            "dmarc_record": "fail",
+            "urgency_level": "high",
+            "confidence_hint": (
+                "AV: attachment is a macro-enabled Excel file (.xlsm), "
+                "macros unsigned"
+            ),
+        },
+        "correct": "QUARANTINE",
+    },
+    {
+        "id": "lv9",
+        "level": "hard",
+        "type": "PHISH",
+        "data": {
+            "sender": "mfa@office365-auth.com",
+            "subject": "Action Required: Update Your MFA Settings Now",
+            "body": (
+                "Your multi-factor authentication settings must be updated. "
+                "Scan the QR code in the attached image to stay compliant."
+            ),
+            "links": [],
+            "has_attachments": True,
+            "spf_record": "softfail",
+            "dmarc_record": "none",
+            "urgency_level": "critical",
+            "confidence_hint": "Gateway: QR code URL resolves to known credential-harvesting page",
+        },
+        "correct": "QUARANTINE",
+    },
+    {
+        "id": "lv10",
+        "level": "hard",
+        "type": "BEC",
+        "data": {
+            "sender": "partner@trusted-firm.com",
+            "subject": "Updated Project Specifications – Download Required by EOD",
+            "body": (
+                "Please find the revised project specs at the link below. "
+                "Deadline is tomorrow morning."
+            ),
+            "links": ["https://trusted-partner.com/files/project_specs_final.zip"],
+            "has_attachments": False,
+            "spf_record": "pass",
+            "dmarc_record": "pass",
+            "urgency_level": "high",
+            "confidence_hint": (
+                "Threat Intel: trusted-firm.com added to IOC feed 6 hours ago "
+                "— possible domain compromise"
+            ),
+        },
+        "correct": "BLOCK_DOMAIN",
+    },
+]
+
+# ── Level → scenario IDs mapping ─────────────────────────────────────────────
+LEVEL_MAP: dict[str, list[str]] = {
+    "easy":   ["lv1", "lv2", "lv3"],
+    "medium": ["lv4", "lv5", "lv6", "lv7"],
+    "hard":   ["lv8", "lv9", "lv10"],
+}
+
+_SCENARIO_BY_ID: dict[str, dict] = {s["id"]: s for s in SCENARIOS}
+
+
+# ═════════════════════════════════════════════════════════════════════════════
+# ENVIRONMENT CLASS
+# ═════════════════════════════════════════════════════════════════════════════
+
+class PhishGuardEnv(OpenEnv):
+    """
+    OpenEnv-compliant simulation environment for SOC analyst LLM benchmarking.
+
+    State
+    -----
+    current_task_idx : int   – pointer into the active (shuffled) scenario list
+    health           : int   – lives remaining (3 → 0)
+    score            : float – cumulative reward for this episode
+    task_scores      : list  – per-step reward history
+    active_level     : str   – current difficulty level
+    scenarios        : list  – scenarios loaded for the current level
+    """
+
+    MAX_HEALTH: int = 3
+
+    def __init__(self) -> None:
+        self.scenarios: List[dict] = []
+        self.current_task_idx: int = 0
+        self.health: int = self.MAX_HEALTH
+        self.score: float = 0.0
+        self.task_scores: List[float] = []
+        self.active_level: str = "easy"
+        # Initialise with easy level so the env is never empty on startup.
+        self._load_level("easy")
+
+    # ── Internal helpers ──────────────────────────────────────────────────────
+
+    def _load_level(self, level: str) -> None:
+        """
+        Filter and shuffle scenarios for the given difficulty level.
+        Resets all state counters.
+
+        Uses SCENARIO_LOADERS from grader.py (mirrors Focus-AI's
+        TASK_LOADERS pattern) so scenario-to-level mapping is
+        centralised in the grader module.
+        """
+        level = level.lower()
+        if level not in LEVEL_MAP:
+            raise ValueError(
+                f"Unknown level '{level}'. Valid choices: easy | medium | hard"
+            )
+        # Use SCENARIO_LOADERS if available, fall back to LEVEL_MAP
+        if level in SCENARIO_LOADERS:
+            ids = SCENARIO_LOADERS[level]()
+        else:
+            ids = LEVEL_MAP[level]
+        subset = [dict(_SCENARIO_BY_ID[sid]) for sid in ids]
+        random.shuffle(subset)
+
+        self.active_level    = level
+        self.scenarios       = subset
+        self.current_task_idx = 0
+        self.health          = self.MAX_HEALTH
+        self.score           = 0.0
+        self.task_scores     = []
+
+    def _is_over(self) -> bool:
+        return self.health <= 0 or self.current_task_idx >= len(self.scenarios)
+
+    # ── Public API ────────────────────────────────────────────────────────────
+
+    def reset(self, level: str = "easy") -> dict:
+        """
+        Reset the environment for a new episode at the given difficulty level.
+
+        Returns the first email observation dict.
+
+        Raises
+        ------
+        ValueError
+            If the level is unknown or maps to zero scenarios.
+        """
+        self._load_level(level)
+        if not self.scenarios:
+            raise ValueError(f"No scenarios found for level '{level}'")
+        first_task = self.scenarios[self.current_task_idx]
+        log.info(
+            "Episode reset | level=%s | first_scenario=%s | total=%d",
+            self.active_level,
+            first_task["id"],
+            len(self.scenarios),
+        )
+        return first_task["data"]
+
+    def step(self, action_str: str) -> tuple:
+        """
+        Advance the simulation by one triage decision.
+
+        Returns
+        -------
+        (obs, reward, done, info)
+        """
+        # ── Guard: episode already over ───────────────────────────────────────
+        if self._is_over():
+            return None, R_BREACH, True, {
+                "task_id":     None,
+                "task_group":  None,
+                "is_correct":  False,
+                "health":      self.health,
+                "feedback":    "Episode already ended. Call /reset to start a new one.",
+                "score":       round(self.score, 4),
+                "task_scores": list(self.task_scores),
+            }
+
+        # ── Resolve current scenario ──────────────────────────────────────────
+        current_task = self.scenarios[self.current_task_idx]
+        task_id      = current_task["id"]
+
+        # ── Grade the action ──────────────────────────────────────────────────
+        reward, verdict_msg = grade_action(
+            action_str,
+            current_task["correct"],
+            current_task["type"],
+        )
+
+        self.score += reward
+        self.task_scores.append(reward)
+
+        log.info(
+            "Step | level=%s | task=%s | action=%s | reward=%.4f | verdict=%s",
+            self.active_level,
+            task_id,
+            action_str.strip().upper(),
+            reward,
+            verdict_msg,
+        )
+
+        # ── Health drain ──────────────────────────────────────────────────────
+        if reward < HEALTH_DRAIN_THRESHOLD:
+            self.health -= 1
+            feedback = (
+                f"⚠️  CRITICAL ERROR: {verdict_msg} "
+                f"| Health remaining: {self.health}/{self.MAX_HEALTH}"
+            )
+        else:
+            feedback = f"✅ Analysis accepted: {verdict_msg}"
+
+        # ── Advance task pointer ──────────────────────────────────────────────
+        done = False
+        self.current_task_idx += 1
+
+        if self.health <= 0:
+            done     = True
+            feedback = "❌ TERMINATED: Too many critical failures — health depleted."
+
+        if self.current_task_idx >= len(self.scenarios):
+            done = True
+            if self.health > 0:
+                feedback = (
+                    f"🏆 SUCCESS: All {len(self.scenarios)} "
+                    f"{self.active_level.upper()} scenarios completed."
+                )
+
+        # ── Next observation ──────────────────────────────────────────────────
+        obs = (
+            self.scenarios[self.current_task_idx]["data"]
+            if not self._is_over()
+            else None
+        )
+
+        return obs, reward, done, {
+            "task_id":     task_id,
+            "task_group":  current_task["level"],
+            "is_correct":  reward >= R_PERFECT,
+            "health":      self.health,
+            "feedback":    feedback,
+            "score":       round(self.score, 4),
+            "task_scores": list(self.task_scores),
+        }
+
+
+# ═════════════════════════════════════════════════════════════════════════════
+# FASTAPI APPLICATION
+# ═════════════════════════════════════════════════════════════════════════════
+
+# Singleton environment instance — shared across all requests.
+_env      = PhishGuardEnv()
+# BUG FIX: asyncio.Lock serialises /reset and /step so concurrent requests
+# cannot race on _env's mutable state (current_task_idx, health, score, etc.).
+_env_lock = asyncio.Lock()
+
+
+@asynccontextmanager
+async def lifespan(app: FastAPI):
+    log.info("PhishGuard-Env %s starting on port 7860.", VERSION)
+    yield
+    log.info("PhishGuard-Env shutting down.")
+
+
+app = FastAPI(
+    title="PhishGuard-Env",
+    description=(
+        "OpenEnv-compliant SOC analyst simulation environment. "
+        "Exposes /reset, /step, /state, and /health for LLM agent benchmarking."
+    ),
+    version=VERSION,
+    lifespan=lifespan,
+)
+
+app.add_middleware(
+    CORSMiddleware,
+    allow_origins=["*"],
+    allow_methods=["GET", "POST"],
+    allow_headers=["*"],
+)
+
+
+# ── Liveness probe ────────────────────────────────────────────────────────────
+@app.get("/health", tags=["Meta"])
+async def health_probe() -> dict:
+    """Liveness probe — HF Spaces and load-balancers call this endpoint."""
+    return {"status": "ok", "env": "PhishGuard-Env", "version": VERSION}
+
+
+# ── Reset ─────────────────────────────────────────────────────────────────────
+@app.post("/reset", tags=["Environment"])
+async def reset(request: Optional[ResetRequest] = None) -> ResetResponse:
+    """
+    Reset the environment for a new episode at the chosen difficulty level.
+
+    Body (optional): { "level": "easy" | "medium" | "hard" }
+    If no body is provided, defaults to "easy".
+    """
+    level = (request.level if request else "easy").lower()
+    if level not in LEVEL_MAP:
+        raise HTTPException(
+            status_code=422,
+            detail=f"Invalid level '{level}'. Must be one of: easy | medium | hard",
+        )
+
+    async with _env_lock:
+        obs = _env.reset(level=level)
+        first_scenario = _env.scenarios[_env.current_task_idx]
+        active_level = _env.active_level
+
+    return ResetResponse(
+        observation=obs,
+        task_id=first_scenario["id"],
+        task_group=first_scenario["level"],
+        level=active_level,
+        # BUG FIX: was len(_env.scenarios) which could be stale;
+        # now reads directly from LEVEL_MAP for the requested level.
+        total_tasks=len(LEVEL_MAP[level]),
+    )
+
+
+# ── Step ──────────────────────────────────────────────────────────────────────
+@app.post("/step", tags=["Environment"])
+async def step(action: PhishAction) -> StepResponse:
+    """
+    Submit one triage action and receive the next observation + reward.
+
+    Body: { "action": "MARK_SAFE | MOVE_TO_SPAM | QUARANTINE | BLOCK_DOMAIN",
+            "reasoning": "optional" }
+    """
+    action_str = action.action.strip().upper()[:64]
+
+    async with _env_lock:
+        obs, reward, done, info = _env.step(action_str)
+
+    return StepResponse(
+        observation=obs,
+        reward=reward,
+        done=done,
+        task_id=info["task_id"],
+        is_correct=info["is_correct"],
+        info=info,
+    )
+
+
+# ── State ────────────────────────────���────────────────────────────────────────
+@app.get("/state", tags=["Environment"])
+async def state() -> dict:
+    """Read-only snapshot of the current environment state."""
+    async with _env_lock:
+        overall = calculate_overall_score(_env.task_scores)
+        return {
+            "level":         _env.active_level,
+            "health":        _env.health,
+            "score":         round(_env.score, 4),
+            "overall_score": overall,
+            "task_index":    _env.current_task_idx,
+            "total_tasks":   len(_env.scenarios),
+            "task_scores":   list(_env.task_scores),
+        }
+
+
+# ── Tasks ─────────────────────────────────────────────────────────────────────
+@app.get("/tasks", tags=["Environment"])
+async def tasks() -> dict:
+    """List all tasks with their IDs, difficulty, and correct actions."""
+    return {
+        "tasks": [
+            {
+                "task_id":    s["id"],
+                "difficulty": s["level"],
+                "type":       s["type"],
+                "correct":    s["correct"],
+            }
+            for s in SCENARIOS
+        ]
+    }
+
+
+# ── Grader ────────────────────────────────────────────────────────────────────
+@app.post("/grader", tags=["Environment"])
+async def grader(request: dict) -> dict:
+    """
+    Grade a triage action for a specific task without running a full episode.
+    The OpenEnv validator calls this endpoint to verify graders are working.
+
+    Body: { "task_id": "lv1", "action": "MOVE_TO_SPAM" }
+    """
+    task_id = request.get("task_id", "lv1")
+    action  = request.get("action",  "QUARANTINE")
+
+    scenario = next((s for s in SCENARIOS if s["id"] == task_id), None)
+    if scenario is None:
+        raise HTTPException(
+            status_code=404,
+            detail=f"Task '{task_id}' not found. Valid IDs: {[s['id'] for s in SCENARIOS]}",
+        )
+
+    reward, message = grade_action(action, scenario["correct"], scenario["type"])
+
+    return {
+        "task_id":    task_id,
+        "action":     action,
+        "reward":     reward,
+        "is_correct": reward >= R_PERFECT,
+        "message":    message,
+    }
+
+
+# ── Grade by Difficulty ───────────────────────────────────────────────────────
+# Mirrors Focus-AI's GRADERS dict pattern — allows grading an entire
+# difficulty level by passing metrics, just like Focus-AI's env.py uses
+# GRADERS[difficulty](metrics) at episode end.
+@app.post("/grade/{difficulty}", tags=["Grading"])
+async def grade_difficulty(difficulty: str, metrics: dict) -> dict:
+    """
+    Grade a full episode for a specific difficulty level using the
+    deterministic grader function.
+
+    This mirrors Focus-AI's GRADERS[difficulty](metrics) pattern.
+
+    Path param: difficulty = easy | medium | hard
+    Body: metrics dict (e.g. {"total_tasks": 3, "correct_actions": 2, ...})
+    """
+    difficulty = difficulty.lower()
+    if difficulty not in GRADERS:
+        raise HTTPException(
+            status_code=422,
+            detail=f"Invalid difficulty '{difficulty}'. Must be one of: {list(GRADERS.keys())}",
+        )
+
+    score = GRADERS[difficulty](metrics)
+    return {
+        "difficulty": difficulty,
+        "score":      score,
+        "metrics":    metrics,
+    }
+
+
+# ── Aggregate Performance Grade ──────────────────────────────────────────────
+@app.post("/grade/performance", tags=["Grading"])
+async def grade_perf(metrics: dict) -> dict:
+    """
+    Cross-difficulty aggregate grader for leaderboard ranking.
+    Mirrors Focus-AI's grade_performance() function.
+    """
+    score = grade_performance(metrics)
+    return {
+        "difficulty": "aggregate",
+        "score":      score,
+        "metrics":    metrics,
+    }
+
+
+# ── Entry point ───────────────────────────────────────────────────────────────
+if __name__ == "__main__":
+    import uvicorn
+    uvicorn.run(
+        "server.app:app",
+        host="0.0.0.0",
+        port=7860,
+        reload=False,
+        log_level="info",
+    )

grader.py

@@ -0,0 +1,403 @@
+"""
+grader.py – PhishGuard-Env SOC Triage Scoring Logic
+====================================================
+
+REWARD CONTRACT  →  OPEN INTERVAL (0.0, 1.0)
+---------------------------------------------
+All rewards are STRICTLY greater than 0 and STRICTLY less than 1.
+The endpoints 0 and 1 are NEVER returned.  This is a hard invariant
+enforced by the constant table below and by calculate_overall_score().
+
+Why open-interval?
+  • 1.0 saturates the leaderboard and implies a theoretically perfect agent.
+  • 0.0 is indistinguishable from a missing data-point in an RL pipeline.
+  • Every decision carries a non-zero gradient signal so training never dies.
+
+REWARD TABLE
+────────────────────────────────────────────────────────────────────────────
+  Constant                 Value   Outcome / Rationale
+  ─────────────────────────────────────────────────────────────────────────
+  R_PERFECT                0.95    Exact match — near-ideal; headroom for 1.0
+  R_MALWARE_QUARANTINE     0.75    MALWARE → QUARANTINE (textbook isolation)
+  R_PHISH_BEC_QUARANTINE   0.60    PHISH/BEC → QUARANTINE (domain still live)
+  R_SPAM_BLOCK             0.40    SPAM → BLOCK_DOMAIN (over-escalation)
+  R_SPAM_QUARANTINE        0.35    SPAM → QUARANTINE (lighter over-escalation)
+  R_WRONG_PROCEDURE        0.10    Wrong action, no direct security/ops harm
+  R_DISRUPTION             0.05    SAFE email blocked — operational cost
+  R_BREACH                 0.02    Threat allowed into inbox — catastrophic
+
+HEALTH-DRAIN THRESHOLD
+────────────────────────────────────────────────────────────────────────────
+  HEALTH_DRAIN_THRESHOLD = 0.15
+  reward < 0.15  →  agent loses one life.
+
+PASS_THRESHOLD
+────────────────────────────────────────────────────────────────────────────
+  PASS_THRESHOLD = 0.50
+
+LEVEL CONTEXT (from env.py)
+────────────────────────────────────────────────────────────────────────────
+  easy   → lv1 (SPAM),    lv2 (PHISH),   lv3 (SAFE)
+  medium → lv4 (MALWARE), lv5 (SAFE),    lv6 (BEC),   lv7 (PHISH)
+  hard   → lv8 (MALWARE), lv9 (PHISH),   lv10 (BEC)
+
+VALID AGENT ACTIONS
+────────────────────────────────────────────────────────────────────────────
+  MARK_SAFE      – deliver to inbox
+  MOVE_TO_SPAM   – bulk / unsolicited mail
+  QUARANTINE     – hold for analyst review
+  BLOCK_DOMAIN   – perimeter block
+
+BUG FIX (v1.0.2 → v1.0.3)
+────────────────────────────────────────────────────────────────────────────
+  SPAM added to _THREAT_TYPES so MARK_SAFE on any threat drains health.
+"""
+
+from __future__ import annotations
+
+import logging
+from typing import Tuple
+
+logger = logging.getLogger(__name__)
+
+__all__ = [
+    "R_PERFECT",
+    "R_MALWARE_QUARANTINE",
+    "R_PHISH_BEC_QUARANTINE",
+    "R_SPAM_BLOCK",
+    "R_SPAM_QUARANTINE",
+    "R_WRONG_PROCEDURE",
+    "R_DISRUPTION",
+    "R_BREACH",
+    "R_PARTIAL",
+    "PASS_THRESHOLD",
+    "HEALTH_DRAIN_THRESHOLD",
+    "grade_action",
+    "grade_easy",
+    "grade_medium",
+    "grade_hard",
+    "grade_performance",
+    "calculate_overall_score",
+    "GRADERS",
+    "SCENARIO_LOADERS",
+]
+
+
+# ══════════════════════════════════════════════════════════════════════════════
+# REWARD CONSTANTS
+# ══════════════════════════════════════════════════════════════════════════════
+
+R_PERFECT              = 0.95
+R_MALWARE_QUARANTINE   = 0.75
+R_PHISH_BEC_QUARANTINE = 0.60
+R_SPAM_BLOCK           = 0.40
+R_SPAM_QUARANTINE      = 0.35
+R_WRONG_PROCEDURE      = 0.10
+R_DISRUPTION           = 0.05
+R_BREACH               = 0.02
+
+R_PARTIAL              = R_MALWARE_QUARANTINE
+PASS_THRESHOLD         = 0.50
+HEALTH_DRAIN_THRESHOLD = 0.15
+
+_THREAT_TYPES  = frozenset({"PHISH", "BEC", "MALWARE", "SPAM"})
+_BLOCKED_MOVES = frozenset({"BLOCK_DOMAIN", "QUARANTINE", "MOVE_TO_SPAM"})
+_VALID_ACTIONS = frozenset({"MARK_SAFE", "MOVE_TO_SPAM", "QUARANTINE", "BLOCK_DOMAIN"})
+
+
+# ══════════════════════════════════════════════════════════════════════════════
+# GRADE_ACTION
+# ══════════════════════════════════════════════════════════════════════════════
+
+def grade_action(
+    agent_output: str,
+    expected_output: str,
+    email_type: str,
+) -> Tuple[float, str]:
+    """
+    Grade one SOC triage decision and return a reward in (0.0, 1.0).
+
+    Parameters
+    ----------
+    agent_output    : Raw action string from the LLM agent (case-insensitive).
+    expected_output : Ground-truth correct action for this scenario.
+    email_type      : Threat category — PHISH | BEC | MALWARE | SPAM | SAFE.
+
+    Returns
+    -------
+    (reward, message)
+        reward  : float strictly in (0.0, 1.0)
+        message : short verdict string for logs and agent feedback.
+    """
+    agent_action    = agent_output.strip().upper()
+    expected_action = expected_output.strip().upper()
+    etype           = email_type.strip().upper()
+
+    # Step 1: Reject unrecognised tokens
+    if agent_action not in _VALID_ACTIONS:
+        return (
+            R_WRONG_PROCEDURE,
+            f"INVALID_ACTION: '{agent_action}' is not a recognised triage action — "
+            f"must be one of: {', '.join(sorted(_VALID_ACTIONS))}",
+        )
+
+    # Step 2: Perfect match
+    if agent_action == expected_action:
+        return R_PERFECT, "PERFECT_TRIAGE: Correct action taken"
+
+    # Step 3: Security Breach
+    if etype in _THREAT_TYPES and agent_action == "MARK_SAFE":
+        return (
+            R_BREACH,
+            f"SECURITY_BREACH: {etype} threat delivered to inbox unimpeded — "
+            "catastrophic failure; source remains active",
+        )
+
+    # Step 4: Business Disruption
+    if etype == "SAFE" and agent_action in _BLOCKED_MOVES:
+        return (
+            R_DISRUPTION,
+            "BUSINESS_DISRUPTION: Legitimate communication was incorrectly blocked — "
+            "operational impact; sender trust degraded",
+        )
+
+    # Step 5: Partial credit
+
+    # MALWARE → QUARANTINE
+    if etype == "MALWARE" and agent_action == "QUARANTINE":
+        return (
+            R_MALWARE_QUARANTINE,
+            "CAUTIOUS: Malware isolated via QUARANTINE — strong containment; "
+            "no further propagation risk detected",
+        )
+
+    # PHISH/BEC → QUARANTINE
+    if etype in {"PHISH", "BEC"} and agent_action == "QUARANTINE":
+        return (
+            R_PHISH_BEC_QUARANTINE,
+            f"UNDER_RESPONSE: {etype} quarantined but source domain still active — "
+            "prefer BLOCK_DOMAIN to prevent further phishing delivery attempts",
+        )
+
+    # SPAM → BLOCK_DOMAIN
+    if etype == "SPAM" and agent_action == "BLOCK_DOMAIN":
+        return (
+            R_SPAM_BLOCK,
+            "OVER_ESCALATION: BLOCK_DOMAIN is disproportionate for SPAM — "
+            "prefer MOVE_TO_SPAM to preserve block-list resources",
+        )
+
+    # SPAM → QUARANTINE
+    if etype == "SPAM" and agent_action == "QUARANTINE":
+        return (
+            R_SPAM_QUARANTINE,
+            "OVER_ESCALATION: QUARANTINE wastes analyst capacity on SPAM — "
+            "prefer MOVE_TO_SPAM for bulk unsolicited mail",
+        )
+
+    # Step 6: Catch-all wrong procedure
+    return (
+        R_WRONG_PROCEDURE,
+        f"INCORRECT_PROCEDURE: '{agent_action}' does not match security policy "
+        f"for {etype} email (expected: {expected_action}) — "
+        "review triage guidelines",
+    )
+
+
+# ══════════════════════════════════════════════════════════════════════════════
+# CALCULATE_OVERALL_SCORE
+# ══════════════════════════════════════════════════════════════════════════════
+
+def calculate_overall_score(task_scores: list) -> float:
+    """
+    Compute the final benchmark score from a list of per-step rewards.
+    Result is clamped to [R_BREACH, R_PERFECT].
+    Empty list returns R_BREACH.
+    """
+    if not task_scores:
+        return R_BREACH
+
+    raw_avg = sum(task_scores) / len(task_scores)
+    clamped = max(R_BREACH, min(R_PERFECT, raw_avg))
+    return round(clamped, 4)
+
+
+# ══════════════════════════════════════════════════════════════════════════════
+# SCORE SAFETY HELPERS
+# ══════════════════════════════════════════════════════════════════════════════
+
+def _safe_score(raw: float) -> float:
+    """
+    Map any float to the open interval (R_BREACH, R_PERFECT).
+
+    Mirrors Focus-AI's safe_score() pattern:
+        safe_score(raw) = LOWER + (UPPER - LOWER) * clamp(raw, 0, 1)
+    where LOWER = R_BREACH (0.02), UPPER = R_PERFECT (0.95).
+
+    Guarantees:
+        raw = 0.0  ->  0.02   (> 0, never equals 0)
+        raw = 1.0  ->  0.95   (< 1, never equals 1)
+    """
+    raw = float(raw)
+    if raw < 0.0:
+        raw = 0.0
+    elif raw > 1.0:
+        raw = 1.0
+    result = R_BREACH + (R_PERFECT - R_BREACH) * raw
+    result = round(result, 6)
+    assert 0.0 < result < 1.0, (
+        f"_safe_score VIOLATION: raw={raw!r} produced result={result!r} "
+        f"which is not strictly inside (0, 1)"
+    )
+    return result
+
+
+def _safe_ratio(num: float, den: float) -> float:
+    """Safe division clamped to [0, 1]."""
+    if den <= 0:
+        return 0.0
+    return max(0.0, min(1.0, num / den))
+
+
+# ══════════════════════════════════════════════════════════════════════════════
+# OPENENV GRADERS
+# Called by the OpenEnv validator — one function per difficulty level.
+# Signature: grade_X(metrics: dict) -> float strictly in (0, 1)
+# ══════════════════════════════════════════════════════════════════════════════
+
+def grade_easy(metrics: dict) -> float:
+    """
+    Grader for easy tasks (lv1-lv3): SPAM, PHISH, SAFE.
+    Scoring: 60% correct action + 40% threat identification accuracy.
+    """
+    total   = max(1, metrics.get("total_tasks", metrics.get("total", 3)))
+    correct = metrics.get("correct_actions", metrics.get("is_correct", 0))
+    on_time = metrics.get("on_time", metrics.get("completed", correct))
+
+    if isinstance(correct, bool):
+        correct = int(correct)
+    if isinstance(on_time, bool):
+        on_time = int(on_time)
+
+    raw = (
+        0.60 * _safe_ratio(correct, total)
+        + 0.40 * _safe_ratio(on_time, total)
+    )
+    return _safe_score(raw)
+
+
+def grade_medium(metrics: dict) -> float:
+    """
+    Grader for medium tasks (lv4-lv7): MALWARE, SAFE HR, BEC, PHISH.
+    Scoring: 40% correct + 35% on-time detection + 25% escalation quality.
+    """
+    total    = max(1, metrics.get("total_tasks", metrics.get("total", 4)))
+    correct  = metrics.get("correct_actions", metrics.get("is_correct", 0))
+    on_time  = metrics.get("on_time", metrics.get("completed", correct))
+    steps    = max(1, metrics.get("total_steps", metrics.get("steps", 4)))
+    good_esc = metrics.get("good_escalation", metrics.get("reward", correct))
+
+    if isinstance(correct,  bool): correct  = int(correct)
+    if isinstance(on_time,  bool): on_time  = int(on_time)
+    if isinstance(good_esc, bool): good_esc = int(good_esc)
+
+    raw = (
+        0.40 * _safe_ratio(correct,  total)
+        + 0.35 * _safe_ratio(on_time,  total)
+        + 0.25 * _safe_ratio(good_esc, steps)
+    )
+    return _safe_score(raw)
+
+
+def grade_hard(metrics: dict) -> float:
+    """
+    Grader for hard tasks (lv8-lv10): MALWARE macro, QR phishing, BEC domain.
+    Scoring: 35% correct + 30% threat accuracy + 20% escalation + 15% priority.
+    """
+    total    = max(1, metrics.get("total_tasks", metrics.get("total", 3)))
+    correct  = metrics.get("correct_actions", metrics.get("is_correct", 0))
+    on_time  = metrics.get("on_time", metrics.get("completed", correct))
+    steps    = max(1, metrics.get("total_steps", metrics.get("steps", 3)))
+    good_esc = metrics.get("good_escalation", metrics.get("reward", correct))
+    hi_pri   = metrics.get("high_priority_correct", correct)
+
+    if isinstance(correct,  bool): correct  = int(correct)
+    if isinstance(on_time,  bool): on_time  = int(on_time)
+    if isinstance(good_esc, bool): good_esc = int(good_esc)
+    if isinstance(hi_pri,   bool): hi_pri   = int(hi_pri)
+
+    raw = (
+        0.35 * _safe_ratio(correct,  total)
+        + 0.30 * _safe_ratio(on_time,  total)
+        + 0.20 * _safe_ratio(good_esc, steps)
+        + 0.15 * _safe_ratio(hi_pri,   max(1, correct))
+    )
+    return _safe_score(raw)
+
+
+def grade_performance(metrics: dict) -> float:
+    """
+    Aggregate grader used for cross-difficulty scoring.
+
+    Mirrors Focus-AI's grade_performance() pattern — provides a single
+    unified score across all difficulty levels for leaderboard ranking.
+
+    Scoring: 40% correct actions + 30% on-time + 20% escalation + 10% priority.
+    """
+    total    = max(1, metrics.get("total_tasks", metrics.get("total", 1)))
+    correct  = metrics.get("correct_actions", metrics.get("is_correct", 0))
+    on_time  = metrics.get("on_time", metrics.get("completed", correct))
+    steps    = max(1, metrics.get("total_steps", metrics.get("steps", 1)))
+    good_esc = metrics.get("good_escalation", metrics.get("reward", correct))
+
+    if isinstance(correct,  bool): correct  = int(correct)
+    if isinstance(on_time,  bool): on_time  = int(on_time)
+    if isinstance(good_esc, bool): good_esc = int(good_esc)
+
+    raw = (
+        0.40 * _safe_ratio(correct,  total)
+        + 0.30 * _safe_ratio(on_time,  total)
+        + 0.20 * _safe_ratio(good_esc, steps)
+        + 0.10 * _safe_ratio(correct,  steps)
+    )
+    return _safe_score(raw)
+
+
+# ══════════════════════════════════════════════════════════════════════════════
+# GRADERS DICT  (mirrors Focus-AI's GRADERS pattern)
+# Maps difficulty level → grader function for easy programmatic lookup.
+# ══════════════════════════════════════════════════════════════════════════════
+
+GRADERS = {
+    "easy":   grade_easy,
+    "medium": grade_medium,
+    "hard":   grade_hard,
+}
+
+
+# ══════════════════════════════════════════════════════════════════════════════
+# SCENARIO LOADERS  (mirrors Focus-AI's TASK_LOADERS pattern)
+# Maps difficulty level → callable that returns the scenario list for that level.
+# Used by env.py to load scenarios without hard-coding level names.
+# ══════════════════════════════════════════════════════════════════════════════
+
+def _get_easy_scenarios() -> list:
+    """Return scenario IDs for easy difficulty."""
+    return ["lv1", "lv2", "lv3"]
+
+
+def _get_medium_scenarios() -> list:
+    """Return scenario IDs for medium difficulty."""
+    return ["lv4", "lv5", "lv6", "lv7"]
+
+
+def _get_hard_scenarios() -> list:
+    """Return scenario IDs for hard difficulty."""
+    return ["lv8", "lv9", "lv10"]
+
+
+SCENARIO_LOADERS = {
+    "easy":   _get_easy_scenarios,
+    "medium": _get_medium_scenarios,
+    "hard":   _get_hard_scenarios,
+}

inference.py

@@ -0,0 +1,323 @@
+"""
+inference.py – PhishGuard-Env Baseline Inference Script
+"""
+
+from __future__ import annotations
+
+import argparse
+import json
+import os
+import sys
+import textwrap
+import time
+from datetime import datetime, timezone
+from typing import Any, Dict, List, Optional
+
+import requests
+from openai import OpenAI
+
+from grader import PASS_THRESHOLD
+
+# ─────────────────────────────────────────────────────────────────────────────
+# Configuration
+# ─────────────────────────────────────────────────────────────────────────────
+
+API_KEY      = os.getenv("HF_TOKEN") or os.getenv("OPENAI_API_KEY")
+API_BASE_URL = os.getenv("API_BASE_URL", "https://router.huggingface.co/v1")
+MODEL_NAME   = os.getenv("MODEL_NAME",   "Qwen/Qwen2.5-72B-Instruct")
+ENV_BASE_URL = os.getenv("ENV_BASE_URL", "http://localhost:7860").rstrip("/")
+
+MAX_STEPS_PER_LEVEL = 15
+HTTP_MAX_RETRIES    = 3
+HTTP_BACKOFF_BASE   = 1.5
+
+# ─────────────────────────────────────────────────────────────────────────────
+# LLM client
+# ─────────────────────────────────────────────────────────────────────────────
+
+if not API_KEY:
+    print("[ERROR] No API key found. Set HF_TOKEN or OPENAI_API_KEY.", flush=True)
+    sys.exit(1)
+
+client = OpenAI(base_url=API_BASE_URL, api_key=API_KEY)
+
+SYSTEM_PROMPT = textwrap.dedent("""
+    You are a SOC (Security Operations Centre) Analyst triaging incoming emails.
+
+    Analyse the email data provided and respond ONLY with valid JSON in this exact format:
+    {"action": "<ACTION>", "reasoning": "<one sentence technical justification>"}
+
+    Valid actions:
+    - MARK_SAFE       : Deliver to inbox (confirmed legitimate email)
+    - MOVE_TO_SPAM    : Bulk/unsolicited mail with no active threat
+    - QUARANTINE      : Hold for analyst review (suspicious but unconfirmed)
+    - BLOCK_DOMAIN    : Block sender domain at perimeter (confirmed phishing/malware source)
+
+    Signal interpretation:
+    - SPF fail + DMARC fail + urgency + links → likely PHISH or MALWARE → BLOCK_DOMAIN or QUARANTINE
+    - Known sender, SPF pass, DMARC pass, no suspicious links → likely SAFE → MARK_SAFE
+    - Bulk unsolicited with no malicious payload → SPAM → MOVE_TO_SPAM
+    - Wire transfer / CEO fraud / financial urgency from unknown domain → BEC → QUARANTINE
+    - Malware attachment confirmed by AV → QUARANTINE (isolate, do not deliver)
+    - Confirmed phishing domain → BLOCK_DOMAIN (sever attack vector)
+
+    confidence_hint field:
+    - This is a contextual signal from your SIEM, mail gateway, or threat-intel feed.
+    - It is intentionally noisy — treat it as one data-point, not ground truth.
+    - If it directly contradicts other signals (SPF, DMARC, links), weigh all evidence.
+""").strip()
+
+
+# ─────────────────────────────────────────────────────────────────────────────
+# HTTP helpers
+# ─────────────────────────────────────────────────────────────────────────────
+
+_session = requests.Session()
+
+
+def _post(endpoint: str, payload: dict) -> dict:
+    url = f"{ENV_BASE_URL}{endpoint}"
+    last_exc: Optional[Exception] = None
+    for attempt in range(HTTP_MAX_RETRIES):
+        try:
+            resp = _session.post(url, json=payload, timeout=30)
+            resp.raise_for_status()
+            return resp.json()
+        except (requests.ConnectionError, requests.Timeout) as exc:
+            last_exc = exc
+            wait = HTTP_BACKOFF_BASE ** attempt
+            print(f"  [WARN] POST {endpoint} failed (attempt {attempt+1}): {exc} — retrying in {wait:.1f}s", flush=True)
+            time.sleep(wait)
+        except requests.HTTPError as exc:
+            if exc.response is not None and exc.response.status_code < 500:
+                raise
+            last_exc = exc
+            wait = HTTP_BACKOFF_BASE ** attempt
+            print(f"  [WARN] POST {endpoint} server error (attempt {attempt+1}): {exc} — retrying in {wait:.1f}s", flush=True)
+            time.sleep(wait)
+    raise RuntimeError(f"POST {endpoint} failed after {HTTP_MAX_RETRIES} attempts: {last_exc}")
+
+
+def _get(endpoint: str) -> dict:
+    url = f"{ENV_BASE_URL}{endpoint}"
+    last_exc: Optional[Exception] = None
+    for attempt in range(HTTP_MAX_RETRIES):
+        try:
+            resp = _session.get(url, timeout=10)
+            resp.raise_for_status()
+            return resp.json()
+        except (requests.ConnectionError, requests.Timeout, requests.HTTPError) as exc:
+            last_exc = exc
+            wait = HTTP_BACKOFF_BASE ** attempt
+            print(f"  [WARN] GET {endpoint} failed (attempt {attempt+1}): {exc} — retrying in {wait:.1f}s", flush=True)
+            time.sleep(wait)
+    raise RuntimeError(f"GET {endpoint} failed after {HTTP_MAX_RETRIES} attempts: {last_exc}")
+
+
+# ─────────────────────────────────────────────────────────────────────────────
+# LLM action selection
+# ─────────────────────────────────────────────────────────────────────────────
+
+def _choose_action(observation: Dict[str, Any]) -> tuple[str, str]:
+    try:
+        completion = client.chat.completions.create(
+            model=MODEL_NAME,
+            messages=[
+                {"role": "system", "content": SYSTEM_PROMPT},
+                {"role": "user",   "content": json.dumps(observation, indent=2)},
+            ],
+            response_format={"type": "json_object"},
+            temperature=0,
+            max_tokens=256,
+        )
+        parsed    = json.loads(completion.choices[0].message.content)
+        action    = parsed.get("action", "QUARANTINE").strip().upper()
+        reasoning = parsed.get("reasoning", "")
+        return action, reasoning
+    except Exception as exc:
+        print(f"  [WARN] LLM error: {exc} — defaulting to QUARANTINE", flush=True)
+        return "QUARANTINE", "Parse error — safe fallback"
+
+
+# ─────────────────────────────────────────────────────────────────────────────
+# Run one level
+# ─────────────────────────────────────────────────────────────────────────────
+
+def run_level(level: str) -> Dict[str, Any]:
+    print(f"\n{'='*60}", flush=True)
+    print(f"  LEVEL: {level.upper()}", flush=True)
+    print(f"{'='*60}", flush=True)
+
+    reset_resp  = _post("/reset", {"level": level})
+    obs         = reset_resp["observation"]
+    total_tasks = reset_resp["total_tasks"]
+    print(f"  Tasks in this level: {total_tasks}", flush=True)
+
+    steps: List[dict] = []
+    step_num  = 0
+    done      = False
+    step_resp: Dict[str, Any] = {}
+
+    while not done and step_num < MAX_STEPS_PER_LEVEL:
+        step_num += 1
+        task_id = reset_resp["task_id"] if step_num == 1 else step_resp.get("task_id", "?")
+        print(f"\n  Step {step_num} | task={task_id}", flush=True)
+
+        action, reasoning = _choose_action(obs)
+        print(f"    -> Action   : {action}", flush=True)
+        print(f"    -> Reasoning: {reasoning[:80]}", flush=True)
+
+        step_resp  = _post("/step", {"action": action, "reasoning": reasoning})
+        reward     = step_resp["reward"]
+        done       = step_resp["done"]
+        is_correct = step_resp["is_correct"]
+        info       = step_resp.get("info", {})
+
+        print(f"    <- Reward   : {reward:.4f}  |  correct={is_correct}  |  done={done}", flush=True)
+        print(f"    <- Feedback : {info.get('feedback', '')[:100]}", flush=True)
+
+        # Structured [STEP] log for validator output parsing
+        print(json.dumps({
+            "[STEP]":     True,
+            "task_id":    step_resp.get("task_id", task_id),
+            "action":     action,
+            "reward":     reward,
+            "is_correct": is_correct,
+        }), flush=True)
+
+        steps.append({
+            "step":       step_num,
+            "task_id":    step_resp.get("task_id", task_id),
+            "action":     action,
+            "reward":     reward,
+            "is_correct": is_correct,
+            "reasoning":  reasoning,
+        })
+
+        obs = step_resp.get("observation")
+        if obs is None and not done:
+            print("  [WARN] obs is None but done=False — breaking.", flush=True)
+            break
+
+    state   = _get("/state")
+    overall = state.get("overall_score", 0.0)
+
+    print(f"\n  {'-'*50}", flush=True)
+    print(f"  Level {level.upper()} complete | steps={step_num} | overall_score={overall:.4f}", flush=True)
+
+    return {
+        "level":         level,
+        "total_tasks":   total_tasks,
+        "steps":         steps,
+        "overall_score": overall,
+    }
+
+
+# ─────────────────────────────────────────────────────────────────────────────
+# Main
+# ─────────────────────────────────────────────────────────────────────────────
+
+def main() -> None:
+    parser = argparse.ArgumentParser(description="PhishGuard-Env Baseline Inference")
+    parser.add_argument(
+        "--level",
+        choices=["easy", "medium", "hard"],
+        default=None,
+        help="Run a single difficulty level instead of all three.",
+    )
+    parser.add_argument(
+        "--output",
+        default=None,
+        help="Path to write JSON results.",
+    )
+    args = parser.parse_args()
+
+    levels_to_run = [args.level] if args.level else ["easy", "medium", "hard"]
+    output_path   = args.output or f"results_{datetime.now(timezone.utc).strftime('%Y%m%dT%H%M%S')}.json"
+
+    # Structured [START] log for validator
+    print(json.dumps({
+        "[START]": True,
+        "model":   MODEL_NAME,
+        "env":     ENV_BASE_URL,
+        "levels":  levels_to_run,
+    }), flush=True)
+
+    try:
+        health = _get("/health")
+        print(f"        server status: {health.get('status', 'unknown')}", flush=True)
+    except Exception as exc:
+        print(f"[ERROR] Cannot reach environment server at {ENV_BASE_URL}: {exc}", flush=True)
+        print("        Make sure `python env.py` is running in another terminal.", flush=True)
+        sys.exit(1)
+
+    results: List[Dict[str, Any]] = []
+    for level in levels_to_run:
+        result = run_level(level)
+        results.append(result)
+        time.sleep(1)
+
+    total_steps   = sum(len(r["steps"])                       for r in results)
+    total_correct = sum(s["is_correct"] for r in results for s in r["steps"])
+    weighted_sum  = sum(r["overall_score"] * r["total_tasks"] for r in results)
+    total_tasks   = sum(r["total_tasks"]                      for r in results)
+    avg_score     = weighted_sum / total_tasks if total_tasks else 0.0
+
+    print(f"\n{'='*60}", flush=True)
+    print(f"  BASELINE SUMMARY", flush=True)
+    print(f"{'='*60}", flush=True)
+    print(f"  Total steps   : {total_steps}", flush=True)
+    print(f"  Correct steps : {total_correct}", flush=True)
+    print(f"  Weighted score: {avg_score:.4f}  (pass threshold: {PASS_THRESHOLD})", flush=True)
+    for r in results:
+        print(f"  {r['level']:8s} score: {r['overall_score']:.4f}  ({r['total_tasks']} tasks)", flush=True)
+
+    success = avg_score >= PASS_THRESHOLD
+
+    # KEY FIX: task_id = level name ("easy" | "medium" | "hard")
+    # This MUST match the task IDs in openenv.yaml so the validator
+    # can find the grader for each task. lv1/lv2/lv3 are NOT in openenv.yaml.
+    all_tasks: List[Dict[str, Any]] = []
+    for r in results:
+        level_correct = sum(1 for s in r["steps"] if s["is_correct"])
+        all_tasks.append({
+            "task_id":    r["level"],
+            "is_correct": level_correct > 0,
+            "reward":     r["overall_score"],
+            "level":      r["level"],
+            "steps":      r["steps"],
+        })
+
+    run_summary = {
+        "timestamp":      datetime.now(timezone.utc).isoformat(),
+        "model":          MODEL_NAME,
+        "env":            ENV_BASE_URL,
+        "levels":         levels_to_run,
+        "total_steps":    total_steps,
+        "total_correct":  total_correct,
+        "avg_score":      round(avg_score, 4),
+        "pass_threshold": PASS_THRESHOLD,
+        "success":        success,
+        "tasks":          all_tasks,
+        "level_results":  results,
+    }
+
+    try:
+        with open(output_path, "w", encoding="utf-8") as fh:
+            json.dump(run_summary, fh, indent=2)
+        print(f"\n  Results saved -> {output_path}", flush=True)
+    except OSError as exc:
+        print(f"\n  [WARN] Could not save results: {exc}", flush=True)
+
+    # Structured [END] log for validator
+    print(json.dumps({
+        "[END]":   True,
+        "success": success,
+        "steps":   total_steps,
+        "score":   round(avg_score, 4),
+        "tasks":   all_tasks,
+    }), flush=True)
+
+
+if __name__ == "__main__":
+    main()

models.py

@@ -0,0 +1,110 @@
+"""
+models.py – PhishGuard-Env Pydantic Models
+==========================================
+
+Typed request / response schemas used by env.py (FastAPI).
+
+PhishAction   : Body schema for POST /step
+StepResponse  : Response schema for POST /step  (OpenEnv grader compliance)
+ResetResponse : Response schema for POST /reset
+
+BUG FIX (v1.0.2 → v1.0.3)
+────────────────────────────────────────────────────────────────────────────
+  StepResponse.task_id was typed as `str` but the episode-already-over guard
+  branch in env.py returns task_id=None.  Pydantic would raise a validation
+  error on every post-episode /step call.
+  Fix: task_id is now Optional[str] with a default of None.
+"""
+
+from __future__ import annotations
+
+from typing import Any, Dict, List, Optional
+
+from pydantic import BaseModel, Field
+
+
+# ─────────────────────────────────────────────────────────────────────────────
+# REQUEST MODELS
+# ─────────────────────────────────────────────────────────────────────────────
+
+class PhishAction(BaseModel):
+    """
+    Action submitted by the agent to POST /step.
+
+    Fields
+    ------
+    action    : One of MARK_SAFE | MOVE_TO_SPAM | QUARANTINE | BLOCK_DOMAIN
+    reasoning : Optional one-sentence technical justification (for logging).
+    """
+    action: str = Field(
+        max_length=64,
+        description="Triage decision. Must be exactly one of: "
+                    "MARK_SAFE | MOVE_TO_SPAM | QUARANTINE | BLOCK_DOMAIN"
+    )
+    reasoning: Optional[str] = Field(
+        default=None,
+        description="One-sentence technical justification for the triage decision",
+    )
+
+
+class ResetRequest(BaseModel):
+    """Body schema for POST /reset."""
+    level: str = Field(
+        default="easy",
+        description="Difficulty level: easy | medium | hard",
+    )
+
+
+# ─────────────────────────────────────────────────────────────────────────────
+# RESPONSE MODELS  (OpenEnv spec — all fields required by validator)
+# ─────────────────────────────────────────────────────────────────────────────
+
+class StepResponse(BaseModel):
+    """
+    Full response for POST /step.
+
+    The OpenEnv validator inspects `task_id` and `is_correct` on every step
+    to count how many distinct tasks have been graded.
+
+    task_id is Optional[str] (not str) because the episode-already-over guard
+    branch returns None — a non-optional field would cause a Pydantic
+    ValidationError on every post-episode call.
+    """
+    observation: Optional[Dict[str, Any]] = Field(
+        description="Next email dict, or null when the episode is done"
+    )
+    reward: float = Field(
+        description="Step reward strictly in (0.0, 1.0)"
+    )
+    done: bool = Field(
+        description="True when all scenarios are complete or health reaches 0"
+    )
+    task_id: Optional[str] = Field(          # BUG FIX: was `str`, must be Optional
+        default=None,
+        description="Scenario ID e.g. 'lv3' — required by OpenEnv validator"
+    )
+    is_correct: bool = Field(
+        description="True when reward >= R_PERFECT (0.95)"
+    )
+    info: Dict[str, Any] = Field(
+        description="Full grader info payload"
+    )
+
+
+class ResetResponse(BaseModel):
+    """Response for POST /reset."""
+    observation: Dict[str, Any] = Field(
+        description="First email observation for this episode"
+    )
+    task_id: str = Field(
+        description="ID of the first scenario in this episode"
+    )
+    task_group: str = Field(
+        description="Difficulty level of the first scenario: easy | medium | hard"
+    )
+    level: str = Field(
+        description="Active difficulty level for this episode"
+    )
+    total_tasks: int = Field(
+        description="Total number of scenarios in this level"
+    )

openenv.yaml

@@ -0,0 +1,87 @@
+# PhishGuard-Env – OpenEnv Manifest
+# Each task must declare a grader so the validator can verify scoring.
+# Pattern mirrors Focus-AI's openenv.yaml with per-difficulty graders + aggregate.
+id: phishguard-env
+name: "PhishGuard-Env — SOC Analyst Phishing Triage"
+description: >
+  A real-world RL environment where an AI agent acts as a SOC analyst
+  triaging phishing, malware, BEC, spam, and safe emails.
+  The agent must classify each email using a four-action triage system.
+  Rewards are graded continuously in (0.0, 1.0).
+
+entry_point: env:app
+
+tasks:
+  - id: easy
+    name: easy_triage
+    grader: "grader:grade_easy"
+    description: >
+      Basic triage — high confidence signals, clear indicators.
+      Covers SPAM detection, basic PHISH blocking, and SAFE email verification.
+      Scoring: 60% correct action + 40% threat identification accuracy.
+
+  - id: medium
+    name: medium_triage
+    grader: "grader:grade_medium"
+    description: >
+      Intermediate triage — mixed signals, some ambiguity.
+      Covers MALWARE quarantine, HR safe emails, BEC wire fraud, shared-doc phishing.
+      Scoring: 40% correct action + 35% on-time detection + 25% escalation quality.
+
+  - id: hard
+    name: hard_triage
+    grader: "grader:grade_hard"
+    description: >
+      Advanced triage — low confidence signals, sophisticated attacks.
+      Covers macro malware, QR-code phishing, BEC domain compromise.
+      Scoring: 35% correct action + 30% threat accuracy + 20% escalation + 15% priority.
+
+  - id: performance
+    name: aggregate_performance
+    grader: "grader:grade_performance"
+    description: >
+      Cross-difficulty aggregate grader for leaderboard ranking.
+      Provides a single unified score across all difficulty levels.
+      Scoring: 40% correct actions + 30% on-time + 20% escalation + 10% priority.
+
+observation_space:
+  type: object
+  fields:
+    sender: string
+    subject: string
+    body: string
+    links: list
+    has_attachments: boolean
+    spf_record: "string — pass | softfail | fail"
+    dmarc_record: "string — pass | fail | none"
+    urgency_level: "string — low | medium | high | critical"
+    confidence_hint: string
+
+action_space:
+  type: string
+  allowed_actions:
+    - MARK_SAFE
+    - MOVE_TO_SPAM
+    - QUARANTINE
+    - BLOCK_DOMAIN
+
+reward:
+  type: continuous
+  range: [0.02, 0.95]
+  description: >
+    All rewards strictly inside open interval (0, 1).
+    R_PERFECT=0.95, R_BREACH=0.02. Health drains on reward < 0.15.
+
+scoring:
+  description: >
+    Each task's final score is STRICTLY inside the open interval (0, 1).
+    The _safe_score() function guarantees: 0.02 ≤ score ≤ 0.95.
+    Minimum achievable: 0.02 (breach / nothing done).
+    Maximum achievable: 0.95 (perfect episode).
+  formula: "_safe_score(raw) = 0.02 + 0.93 * clamp(raw, 0.0, 1.0)"
+
+tags:
+  - cybersecurity
+  - openenv
+  - phishing
+  - soc

pyproject.toml

@@ -0,0 +1,27 @@
+[build-system]
+requires = ["setuptools>=61.0", "wheel"]
+build-backend = "setuptools.build_meta"
+
+[project]
+name = "phishguard-env"
+version = "1.0.3"
+description = "A SOC Analyst simulation environment for phishing email triage."
+readme = "README.md"
+requires-python = ">=3.10"
+
+dependencies = [
+    "openenv",
+    "pydantic>=2.0,<3.0",
+    "fastapi>=0.111,<1.0",
+    "uvicorn[standard]>=0.29,<1.0",
+    "openai>=1.30,<2.0",
+    "requests>=2.31,<3.0",
+    "python-dotenv>=1.0,<2.0",
+]
+
+[project.scripts]
+phishguard-server = "server.app:main"
+
+[project.urls]
+Homepage = "https://huggingface.co/spaces/Scalar-hackathon/Phishing-env"
+Repository = "https://github.com/og-arin/phishguard-env"

requirements.txt

@@ -0,0 +1,18 @@
+# PhishGuard-Env dependencies
+# Pinned to minor versions for reproducibility.
+# Update pins deliberately — do not let pip float to latest.
+
+# ── Environment server ────────────────────────────────────────────────────────
+fastapi>=0.111,<1.0
+uvicorn[standard]>=0.29,<1.0
+pydantic>=2.0,<3.0
+
+# ── OpenEnv framework ─────────────────────────────────────────────────────────
+openenv
+
+# ── Inference agent ───────────────────────────────────────────────────────────
+openai>=1.30,<2.0
+requests>=2.31,<3.0       # BUG FIX: was missing; inference.py imports requests
+
+# ── Utilities ─────────────────────────────────────────────────────────────────
+python-dotenv>=1.0,<2.0

server/__init__.py

@@ -0,0 +1 @@
+# server package — makes `server.app` importable as a module.

server/app.py

@@ -0,0 +1,28 @@
+"""
+server/app.py - OpenEnv Validator Entry Point
+"""
+
+import sys
+import os
+
+sys.path.insert(0, os.path.dirname(os.path.dirname(os.path.abspath(__file__))))
+
+from env import app
+
+__all__ = ["app"]
+
+
+def main():
+    """Entry point for [project.scripts] and direct execution."""
+    import uvicorn
+    uvicorn.run(
+        "server.app:app",
+        host="0.0.0.0",
+        port=7860,
+        reload=False,
+        log_level="info",
+    )
+
+
+if __name__ == "__main__":
+    main()

test_api.py

@@ -0,0 +1,211 @@
+"""
+test_api.py – Integration tests for FastAPI endpoints
+======================================================
+
+Run with:  pytest test_api.py -v
+
+Tests the four endpoints (/health, /reset, /step, /state) using FastAPI's
+TestClient (synchronous wrapper around httpx).  No external LLM or
+network calls are needed — these hit the in-process ASGI app directly.
+
+Coverage
+--------
+  • GET  /health  → 200, fields present
+  • POST /reset   → 200 with body, 200 without body, 422 on bad level
+  • POST /step    → 200, reward in (0,1), done flag, episode-over guard
+  • GET  /state   → 200, expected keys present
+"""
+
+from __future__ import annotations
+
+import pytest
+from fastapi.testclient import TestClient
+
+from env import app
+
+
+# ─────────────────────────────────────────────────────────────────────────────
+# Fixtures
+# ─────────────────────────────────────────────────────────────────────────────
+
+@pytest.fixture()
+def client():
+    """Yield a fresh TestClient; environment state is shared (singleton)."""
+    with TestClient(app) as c:
+        yield c
+
+
+# ═════════════════════════════════════════════════════════════════════════════
+# GET /health
+# ═════════════════════════════════════════════════════════════════════════════
+
+class TestHealth:
+    def test_health_returns_200(self, client):
+        resp = client.get("/health")
+        assert resp.status_code == 200
+
+    def test_health_has_status_ok(self, client):
+        data = client.get("/health").json()
+        assert data["status"] == "ok"
+
+    def test_health_has_version(self, client):
+        data = client.get("/health").json()
+        assert "version" in data
+
+
+# ═════════════════════════════════════════════════════════════════════════════
+# POST /reset
+# ═════════════════════════════════════════════════════════════════════════════
+
+class TestReset:
+    def test_reset_with_body(self, client):
+        resp = client.post("/reset", json={"level": "easy"})
+        assert resp.status_code == 200
+        data = resp.json()
+        assert "observation" in data
+        assert "task_id" in data
+        assert data["level"] == "easy"
+        assert data["total_tasks"] == 3
+
+    def test_reset_without_body(self, client):
+        """The OpenEnv validator sends POST /reset with no body."""
+        resp = client.post("/reset")
+        assert resp.status_code == 200
+        data = resp.json()
+        assert data["level"] == "easy"
+        assert "observation" in data
+
+    def test_reset_medium(self, client):
+        resp = client.post("/reset", json={"level": "medium"})
+        assert resp.status_code == 200
+        data = resp.json()
+        assert data["level"] == "medium"
+        assert data["total_tasks"] == 4
+
+    def test_reset_hard(self, client):
+        resp = client.post("/reset", json={"level": "hard"})
+        assert resp.status_code == 200
+        data = resp.json()
+        assert data["level"] == "hard"
+        assert data["total_tasks"] == 3
+
+    def test_reset_invalid_level(self, client):
+        resp = client.post("/reset", json={"level": "nightmare"})
+        assert resp.status_code == 422
+
+    def test_reset_returns_observation_fields(self, client):
+        data = client.post("/reset", json={"level": "easy"}).json()
+        obs = data["observation"]
+        assert "sender" in obs
+        assert "subject" in obs
+        assert "body" in obs
+        assert "spf_record" in obs
+
+    def test_reset_returns_task_id(self, client):
+        data = client.post("/reset", json={"level": "easy"}).json()
+        assert data["task_id"].startswith("lv")
+
+    def test_reset_returns_task_group(self, client):
+        data = client.post("/reset", json={"level": "easy"}).json()
+        assert data["task_group"] == "easy"
+
+
+# ═════════════════════════════════════════════════════════════════════════════
+# POST /step
+# ═════════════════════════���═══════════════════════════════════════════════════
+
+class TestStep:
+    def test_step_returns_200(self, client):
+        client.post("/reset", json={"level": "easy"})
+        resp = client.post("/step", json={"action": "MARK_SAFE"})
+        assert resp.status_code == 200
+
+    def test_step_has_required_fields(self, client):
+        client.post("/reset", json={"level": "easy"})
+        data = client.post("/step", json={"action": "QUARANTINE"}).json()
+        assert "observation" in data
+        assert "reward" in data
+        assert "done" in data
+        assert "task_id" in data
+        assert "is_correct" in data
+        assert "info" in data
+
+    def test_step_reward_in_open_interval(self, client):
+        client.post("/reset", json={"level": "easy"})
+        data = client.post("/step", json={"action": "MARK_SAFE"}).json()
+        assert 0.0 < data["reward"] < 1.0
+
+    def test_step_with_reasoning(self, client):
+        client.post("/reset", json={"level": "easy"})
+        resp = client.post("/step", json={
+            "action": "QUARANTINE",
+            "reasoning": "Suspicious sender domain",
+        })
+        assert resp.status_code == 200
+
+    def test_full_easy_episode(self, client):
+        """Run all 3 easy tasks and verify done=True at the end."""
+        client.post("/reset", json={"level": "easy"})
+        done = False
+        steps = 0
+        while not done and steps < 10:
+            data = client.post("/step", json={"action": "QUARANTINE"}).json()
+            done = data["done"]
+            steps += 1
+        assert done is True
+        assert steps <= 5  # easy has 3 tasks; should never exceed that
+
+    def test_step_after_episode_done(self, client):
+        """Steps after episode ends should return done=True gracefully."""
+        client.post("/reset", json={"level": "easy"})
+        # Exhaust all tasks
+        for _ in range(5):
+            resp = client.post("/step", json={"action": "QUARANTINE"})
+        # Extra step after episode is over
+        data = client.post("/step", json={"action": "MARK_SAFE"}).json()
+        assert data["done"] is True
+
+    def test_step_invalid_action_still_200(self, client):
+        """Invalid actions are graded as WRONG_PROCEDURE, not rejected with 4xx."""
+        client.post("/reset", json={"level": "easy"})
+        resp = client.post("/step", json={"action": "DELETE_EVERYTHING"})
+        assert resp.status_code == 200
+        data = resp.json()
+        assert data["reward"] == 0.10  # R_WRONG_PROCEDURE
+
+
+# ═════════════════════════════════════════════════════════════════════════════
+# GET /state
+# ═════════════════════════════════════════════════════════════════════════════
+
+class TestState:
+    def test_state_returns_200(self, client):
+        resp = client.get("/state")
+        assert resp.status_code == 200
+
+    def test_state_has_expected_keys(self, client):
+        client.post("/reset", json={"level": "easy"})
+        data = client.get("/state").json()
+        assert "active_level" in data
+        assert "current_task_idx" in data
+        assert "health" in data
+        assert "score" in data
+        assert "task_scores" in data
+        assert "scenarios_total" in data
+        assert "overall_score" in data
+
+    def test_state_after_reset(self, client):
+        client.post("/reset", json={"level": "medium"})
+        data = client.get("/state").json()
+        assert data["active_level"] == "medium"
+        assert data["health"] == 3
+        assert data["score"] == 0.0
+        assert data["task_scores"] == []
+        assert data["current_task_idx"] == 0
+
+    def test_state_after_step(self, client):
+        client.post("/reset", json={"level": "easy"})
+        client.post("/step", json={"action": "QUARANTINE"})
+        data = client.get("/state").json()
+        assert data["current_task_idx"] == 1
+        assert len(data["task_scores"]) == 1

test_grader.py

@@ -0,0 +1,264 @@
+"""
+test_grader.py – Unit tests for grader.py
+==========================================
+
+Run with:  pytest test_grader.py -v
+
+These tests cover every branch of grade_action() and the edge-cases of
+calculate_overall_score().  They also act as a regression guard — any
+change to a reward constant will immediately fail the assertion that was
+relying on it, forcing an intentional review.
+"""
+
+from __future__ import annotations
+
+import pytest
+
+from grader import (
+    HEALTH_DRAIN_THRESHOLD,
+    PASS_THRESHOLD,
+    R_BREACH,
+    R_DISRUPTION,
+    R_MALWARE_QUARANTINE,
+    R_PARTIAL,
+    R_PERFECT,
+    R_PHISH_BEC_QUARANTINE,
+    R_SPAM_BLOCK,
+    R_SPAM_QUARANTINE,
+    R_WRONG_PROCEDURE,
+    calculate_overall_score,
+    grade_action,
+)
+
+
+# ═════════════════════════════════════════════════════════════════════════════
+# CONSTANT SANITY CHECKS
+# ═════════════════════════════════════════════════════════════════════════════
+
+class TestRewardConstants:
+    def test_all_rewards_in_open_interval(self):
+        for r in [R_PERFECT, R_MALWARE_QUARANTINE, R_PHISH_BEC_QUARANTINE,
+                  R_SPAM_BLOCK, R_SPAM_QUARANTINE, R_WRONG_PROCEDURE,
+                  R_DISRUPTION, R_BREACH]:
+            assert 0.0 < r < 1.0, f"Reward {r} violates open-interval contract"
+
+    def test_reward_ordering(self):
+        assert R_BREACH < R_DISRUPTION < R_WRONG_PROCEDURE
+        assert R_WRONG_PROCEDURE < HEALTH_DRAIN_THRESHOLD
+        assert HEALTH_DRAIN_THRESHOLD < R_SPAM_QUARANTINE
+        assert R_SPAM_QUARANTINE < R_SPAM_BLOCK
+        assert R_SPAM_BLOCK < R_PHISH_BEC_QUARANTINE
+        assert R_PHISH_BEC_QUARANTINE < R_MALWARE_QUARANTINE
+        assert R_MALWARE_QUARANTINE < R_PERFECT
+
+    def test_partial_alias(self):
+        assert R_PARTIAL == R_MALWARE_QUARANTINE
+
+    def test_health_drain_covers_breach_disruption_wrong(self):
+        assert R_BREACH < HEALTH_DRAIN_THRESHOLD
+        assert R_DISRUPTION < HEALTH_DRAIN_THRESHOLD
+        assert R_WRONG_PROCEDURE < HEALTH_DRAIN_THRESHOLD
+
+    def test_cautious_scores_never_drain_health(self):
+        assert R_SPAM_QUARANTINE >= HEALTH_DRAIN_THRESHOLD
+        assert R_SPAM_BLOCK >= HEALTH_DRAIN_THRESHOLD
+        assert R_PHISH_BEC_QUARANTINE >= HEALTH_DRAIN_THRESHOLD
+        assert R_MALWARE_QUARANTINE >= HEALTH_DRAIN_THRESHOLD
+
+
+# ═════════════════════════════════════════════════════════════════════════════
+# GRADE_ACTION — PERFECT MATCH
+# ═════════════════════════════════════════════════════════════════════════════
+
+class TestPerfectTriage:
+    @pytest.mark.parametrize("action,etype", [
+        ("MOVE_TO_SPAM",  "SPAM"),
+        ("BLOCK_DOMAIN",  "PHISH"),
+        ("MARK_SAFE",     "SAFE"),
+        ("QUARANTINE",    "MALWARE"),
+        ("QUARANTINE",    "BEC"),
+        ("BLOCK_DOMAIN",  "BEC"),
+        ("QUARANTINE",    "PHISH"),
+    ])
+    def test_perfect_match(self, action, etype):
+        reward, msg = grade_action(action, action, etype)
+        assert reward == R_PERFECT
+        assert "PERFECT_TRIAGE" in msg
+
+    def test_case_insensitive_action(self):
+        reward, _ = grade_action("move_to_spam", "MOVE_TO_SPAM", "SPAM")
+        assert reward == R_PERFECT
+
+    def test_case_insensitive_type(self):
+        reward, _ = grade_action("MARK_SAFE", "MARK_SAFE", "safe")
+        assert reward == R_PERFECT
+
+
+# ═════════════════════════════════════════════════════════════════════════════
+# GRADE_ACTION — SECURITY BREACH
+# ═════════════════════════════════════════════════════════════════════════════
+
+class TestSecurityBreach:
+    @pytest.mark.parametrize("etype", ["PHISH", "BEC", "MALWARE", "SPAM"])
+    def test_mark_safe_on_any_threat_is_breach(self, etype):
+        """BUG FIX coverage: SPAM + MARK_SAFE must return R_BREACH (not R_WRONG_PROCEDURE)."""
+        reward, msg = grade_action("MARK_SAFE", "BLOCK_DOMAIN", etype)
+        assert reward == R_BREACH
+        assert "SECURITY_BREACH" in msg
+
+    def test_spam_mark_safe_is_breach_not_wrong_procedure(self):
+        """Regression: the original code excluded SPAM from _THREAT_TYPES."""
+        reward, _ = grade_action("MARK_SAFE", "MOVE_TO_SPAM", "SPAM")
+        assert reward == R_BREACH, (
+            "SPAM + MARK_SAFE must be R_BREACH — SPAM was excluded from "
+            "_THREAT_TYPES in the v1.0.2 bug"
+        )
+        assert reward < HEALTH_DRAIN_THRESHOLD  # must drain health
+
+
+# ═════════════════════════════════════════════════════════════════════════════
+# GRADE_ACTION — BUSINESS DISRUPTION
+# ═════════════════════════════════════════════════════════════════════════════
+
+class TestBusinessDisruption:
+    @pytest.mark.parametrize("blocking_action", ["BLOCK_DOMAIN", "QUARANTINE", "MOVE_TO_SPAM"])
+    def test_blocking_safe_email_is_disruption(self, blocking_action):
+        reward, msg = grade_action(blocking_action, "MARK_SAFE", "SAFE")
+        assert reward == R_DISRUPTION
+        assert "BUSINESS_DISRUPTION" in msg
+
+
+# ═════════════════════════════════════════════════════════════════════════════
+# GRADE_ACTION — PARTIAL CREDIT
+# ═════════════════════════════════════════════════════════════════════════════
+
+class TestPartialCredit:
+    def test_malware_quarantine(self):
+        reward, msg = grade_action("QUARANTINE", "BLOCK_DOMAIN", "MALWARE")
+        assert reward == R_MALWARE_QUARANTINE
+        assert "CAUTIOUS" in msg
+
+    def test_phish_quarantine(self):
+        reward, msg = grade_action("QUARANTINE", "BLOCK_DOMAIN", "PHISH")
+        assert reward == R_PHISH_BEC_QUARANTINE
+        assert "UNDER_RESPONSE" in msg
+
+    def test_bec_quarantine(self):
+        reward, msg = grade_action("QUARANTINE", "BLOCK_DOMAIN", "BEC")
+        assert reward == R_PHISH_BEC_QUARANTINE
+
+    def test_spam_block_domain(self):
+        reward, msg = grade_action("BLOCK_DOMAIN", "MOVE_TO_SPAM", "SPAM")
+        assert reward == R_SPAM_BLOCK
+        assert "OVER_ESCALATION" in msg
+
+    def test_spam_quarantine(self):
+        reward, msg = grade_action("QUARANTINE", "MOVE_TO_SPAM", "SPAM")
+        assert reward == R_SPAM_QUARANTINE
+        assert "OVER_ESCALATION" in msg
+
+    @pytest.mark.parametrize("action", ["QUARANTINE", "BLOCK_DOMAIN", "MOVE_TO_SPAM"])
+    def test_partial_credits_never_drain_health(self, action):
+        reward, _ = grade_action(action, "MOVE_TO_SPAM", "SPAM")
+        assert reward >= HEALTH_DRAIN_THRESHOLD
+
+
+# ═════════════════════════════════════════════════════════════════════════════
+# GRADE_ACTION — WRONG PROCEDURE / INVALID
+# ═════════════════════════════════════════════════════════════════════════════
+
+class TestWrongProcedure:
+    def test_malware_block_domain_is_wrong_procedure(self):
+        """lv4 / lv8: expected QUARANTINE, agent picks BLOCK_DOMAIN."""
+        reward, msg = grade_action("BLOCK_DOMAIN", "QUARANTINE", "MALWARE")
+        assert reward == R_WRONG_PROCEDURE
+        assert "INCORRECT_PROCEDURE" in msg
+
+    def test_invalid_action_token(self):
+        reward, msg = grade_action("DELETE", "QUARANTINE", "MALWARE")
+        assert reward == R_WRONG_PROCEDURE
+        assert "INVALID_ACTION" in msg
+
+    def test_invalid_action_drains_health(self):
+        reward, _ = grade_action("NONSENSE", "QUARANTINE", "PHISH")
+        assert reward < HEALTH_DRAIN_THRESHOLD
+
+
+# ═════════════════════════════════════════════════════════════════════════════
+# CALCULATE_OVERALL_SCORE
+# ═════════════════════════════════════════════════════════════════════════════
+
+class TestCalculateOverallScore:
+    def test_empty_list_returns_r_breach(self):
+        assert calculate_overall_score([]) == R_BREACH
+
+    def test_all_perfect_returns_r_perfect(self):
+        assert calculate_overall_score([R_PERFECT, R_PERFECT, R_PERFECT]) == R_PERFECT
+
+    def test_result_never_exceeds_r_perfect(self):
+        assert calculate_overall_score([1.0, 1.0]) == R_PERFECT
+
+    def test_result_never_below_r_breach(self):
+        assert calculate_overall_score([0.0, 0.0]) == R_BREACH
+
+    def test_easy_all_perfect(self):
+        scores = [R_PERFECT] * 3
+        assert calculate_overall_score(scores) == R_PERFECT
+
+    def test_medium_mixed(self):
+        scores = [R_PERFECT, R_BREACH, R_PERFECT, R_PHISH_BEC_QUARANTINE]
+        result = calculate_overall_score(scores)
+        expected = round((R_PERFECT + R_BREACH + R_PERFECT + R_PHISH_BEC_QUARANTINE) / 4, 4)
+        assert result == expected
+
+    def test_hard_mostly_bad(self):
+        scores = [R_BREACH, R_DISRUPTION, R_PERFECT]
+        result = calculate_overall_score(scores)
+        assert result < PASS_THRESHOLD
+
+    def test_returns_four_decimal_places(self):
+        result = calculate_overall_score([R_PERFECT, R_BREACH])
+        assert result == round(result, 4)
+
+    def test_single_perfect_step(self):
+        assert calculate_overall_score([R_PERFECT]) == R_PERFECT
+
+    def test_single_breach_step(self):
+        assert calculate_overall_score([R_BREACH]) == R_BREACH
+
+
+# ═════════════════════════════════════════════════════════════════════════════
+# EDGE CASES — INPUT NORMALIZATION
+# ═════════════════════════════════════════════════════════════════════════════
+
+class TestInputNormalization:
+    def test_whitespace_padded_action(self):
+        """Actions with leading/trailing spaces should still match."""
+        reward, msg = grade_action("  MARK_SAFE  ", "MARK_SAFE", "SAFE")
+        assert reward == R_PERFECT
+        assert "PERFECT_TRIAGE" in msg
+
+    def test_mixed_case_email_type_phish(self):
+        reward, _ = grade_action("MARK_SAFE", "BLOCK_DOMAIN", "Phish")
+        assert reward == R_BREACH
+
+    def test_mixed_case_email_type_bec(self):
+        reward, _ = grade_action("QUARANTINE", "BLOCK_DOMAIN", "bEc")
+        assert reward == R_PHISH_BEC_QUARANTINE
+
+    def test_tab_in_action(self):
+        reward, _ = grade_action("\tQUARANTINE\t", "QUARANTINE", "MALWARE")
+        assert reward == R_PERFECT
+
+    def test_empty_action_string(self):
+        reward, msg = grade_action("", "QUARANTINE", "MALWARE")
+        assert reward == R_WRONG_PROCEDURE
+        assert "INVALID_ACTION" in msg
+
+    def test_calculate_overall_score_single_breach(self):
+        assert calculate_overall_score([R_BREACH]) == R_BREACH
+
+    def test_calculate_overall_score_all_disruption(self):
+        result = calculate_overall_score([R_DISRUPTION] * 5)
+        assert result == R_DISRUPTION
+