import time
NB_START_TIME = time.time()
TRAIN_MODEL = "qwen-0.5b"  
MAX_STEPS          = 400  # Defender GRPO steps
ATTACKER_STEPS     = 400   # Attacker GRPO steps
PHASE_ALTERNATIONS = 1     # One attacker phase + one defender phase

NUM_GENERATIONS    = 4
LEARNING_RATE      = 2e-6
BATCH_SIZE         = 1
GRAD_ACCUMULATION  = 2
MAX_COMPLETION_LEN = 32    # Smoke-test: action-only output

SEED_EPISODES      = 256   # Prompt dataset size per phase
REFRESH_EPISODES   = 256    # Smaller refresh for smoke test

# --- LLM-as-Judge (Groq) [disabled] ---
GROQ_API_KEY = ""  
GROQ_MODEL   = "llama-3.1-8b-instant"
JUDGE_PROB   = 0.0

EVAL_EPISODES  = 50

HF_TOKEN    = "hf_<REDACTED>"           l

MODEL_REGISTRY = {
    "qwen-0.5b": "unsloth/Qwen2.5-0.5B-Instruct",
    "llama-1b":  "unsloth/Llama-3.2-1B-Instruct",
    "qwen-1.5b": "unsloth/Qwen2.5-1.5B-Instruct"
}

TRAIN_MODEL_ID = MODEL_REGISTRY[TRAIN_MODEL]
BASELINE_MODEL = TRAIN_MODEL_ID
print(f"Training model: {TRAIN_MODEL_ID}")
print(f"GRPO steps: attacker={ATTACKER_STEPS}, defender={MAX_STEPS}")
print(f"Groq judge enabled: {bool(GROQ_API_KEY)} | prob={JUDGE_PROB}")

Training model: unsloth/Qwen2.5-0.5B-Instruct
GRPO steps: attacker=400, defender=400
Groq judge enabled: False | prob=0.0

%%capture
import os
os.environ["UNSLOTH_VLLM_STANDBY"] = "1"

!pip install --upgrade -qqq uv

try: import numpy, PIL; _numpy = f'numpy=={numpy.__version__}'; _pil = f'pillow=={PIL.__version__}'
except: _numpy = "numpy"; _pil = "pillow"

try: import subprocess; is_t4 = "Tesla T4" in str(subprocess.check_output(["nvidia-smi"]))
except: is_t4 = False

_vllm, _triton = ('vllm==0.9.2', 'triton==3.2.0') if is_t4 else ('vllm==0.15.1', 'triton')
!uv pip install -qqq --upgrade {_vllm} {_numpy} {_pil} bitsandbytes xformers unsloth
!uv pip install -qqq {_triton}
!uv pip install transformers==4.56.2
!uv pip install --no-deps trl==0.22.2      # --no-deps is critical: skips mergekit entirely
!uv pip install -qqq matplotlib numpy datasets groq
!pip uninstall -y torchcodec 2>/dev/null; echo "torchcodec removed"

# APIs removed for 10x training speed.
# We use a deterministic Cyber Warfare environment.

# ============================================================
# Patch-or-Exploit Duel — MULTI-STEP (up to 3 steps per episode)
# ============================================================
# Design goals:
# - 3-step episodes → agent must reason about consequences, not just react
# - Health state evolves across steps → genuine partial observability
# - Attack chains: successful breach in step 1 unlocks follow-on attacks
# - Rolling-window memory → agents adapt to each other over time
# - Per-scenario correct defense → breaks global lookup table

import random
import re
from collections import Counter, deque

ATTACKS  = [
    "PHISH", "BRUTEFORCE", "DRIVEBY", "RANSOM",
    "SQLI",  "RCE",        "LPE",     "SUPPLYCHAIN",
]
DEFENSES = [
    "MFA", "PATCH", "EDR", "BACKUP",
    "WAF", "LEASTPRIV", "SBOM", "ROTATEKEYS",
]

EPISODE_STEPS = 3   # steps per episode

# Attack chains: if attack X breaches, follow-on attacks become available next step
ATTACK_CHAINS = {
    "PHISH":      ["LPE", "RANSOM"],       # credential theft → privilege escalation or ransomware
    "BRUTEFORCE": ["RANSOM", "DRIVEBY"],   # access → ransomware or drive-by pivot
    "DRIVEBY":    ["RCE", "LPE"],          # browser exploit → remote code exec or priv esc
    "SQLI":       ["RCE", "LPE"],          # DB injection → code execution
    "RCE":        ["RANSOM", "SUPPLYCHAIN"],
    "LPE":        ["RANSOM", "SUPPLYCHAIN"],
    "RANSOM":     [],
    "SUPPLYCHAIN":[],
}

SCENARIOS = [
    # ── PHISH pair ────────────────────────────────────────────────────────────
    {
        "id": "bulk_phish",
        "weakness": "PHISH",
        "hint": "Log: bulk phishing campaign detected, no clicks yet.",
        "profile": "Org: 500 users, MFA=available",
        "counter": "MFA",
    },
    {
        "id": "exec_phish",
        "weakness": "PHISH",
        "hint": "Log: CFO clicked spear-phish link, credential harvester active.",
        "profile": "Target: executive, EDR=installed, MFA=bypassed",
        "counter": "EDR",
    },
    # ── BRUTEFORCE pair ───────────────────────────────────────────────────────
    {
        "id": "ssh_brute",
        "weakness": "BRUTEFORCE",
        "hint": "Log: many failed SSH logins from one IP, then a success.",
        "profile": "Service: ssh, rate_limits=off, MFA=available",
        "counter": "MFA",
    },
    {
        "id": "api_brute",
        "weakness": "BRUTEFORCE",
        "hint": "Log: API key rotation overdue, brute-force on API gateway.",
        "profile": "Service: REST API, MFA=not_applicable, keys=stale",
        "counter": "ROTATEKEYS",
    },
    # ── DRIVEBY pair ──────────────────────────────────────────────────────────
    {
        "id": "unpatched_browser",
        "weakness": "DRIVEBY",
        "hint": "Log: drive-by ad redirect chain observed on endpoints.",
        "profile": "Endpoints: outdated browser, EDR=absent",
        "counter": "PATCH",
    },
    {
        "id": "driveby_edr",
        "weakness": "DRIVEBY",
        "hint": "Log: drive-by payload dropped, C2 beacon attempting to run.",
        "profile": "Endpoints: browser patched, EDR=installed",
        "counter": "EDR",
    },
    # ── RANSOM pair ───────────────────────────────────────────────────────────
    {
        "id": "backup_gap",
        "weakness": "RANSOM",
        "hint": "Log: backups failed 3 days in a row, encryption starting.",
        "profile": "Backups: not tested, EDR=absent",
        "counter": "BACKUP",
    },
    {
        "id": "ransom_edr",
        "weakness": "RANSOM",
        "hint": "Log: ransomware binary detected in staging, not yet executed.",
        "profile": "Backups: healthy, EDR=installed",
        "counter": "EDR",
    },
]

def get_counter(sc: dict) -> str:
    return sc.get("counter", COUNTER_FALLBACK.get(sc["weakness"], "PATCH"))

COUNTER_FALLBACK = {
    "PHISH": "MFA",   "BRUTEFORCE": "MFA",  "DRIVEBY": "PATCH",
    "RANSOM": "BACKUP", "SQLI": "WAF",      "RCE": "PATCH",
    "LPE": "LEASTPRIV", "SUPPLYCHAIN": "SBOM",
}

# Rolling-window memory
MEMORY_WINDOW   = 10
recent_attacks  = deque(maxlen=MEMORY_WINDOW)
recent_defenses = deque(maxlen=MEMORY_WINDOW)
recent_breaches = deque(maxlen=MEMORY_WINDOW)

def _summary(prefix: str, items, universe):
    c     = Counter(items)
    parts = [f"{u}={c.get(u, 0)}" for u in universe]
    return f"{prefix}({len(items)}): " + " ".join(parts)

def attacker_memory_line():
    return _summary("Defender_recent", list(recent_defenses), DEFENSES)

def defender_memory_lines():
    return (
        _summary("Attacker_recent", list(recent_attacks),  ATTACKS)
        + "\n"
        + _summary("Recent_breaches", list(recent_breaches), ATTACKS)
    )

# ── Episode state (multi-step) ────────────────────────────────────────────────
class EpisodeState:
    """Tracks health and history within a 3-step episode.
    Passed through prompt so Defender sees consequences of prior steps."""
    def __init__(self, sc):
        self.sc         = sc
        self.health     = 1.0
        self.step       = 0
        self.history    = []    # list of (attack, defense, breached)
        self.unlocked   = []    # follow-on attacks unlocked by prior breaches

    def status(self):
        if self.health >= 0.8:   return "STABLE"
        if self.health >= 0.5:   return "DEGRADED"
        return "CRITICAL"

    def apply(self, attack, defense, breached):
        if breached:
            damage = {"RANSOM": 0.4, "RCE": 0.35, "SUPPLYCHAIN": 0.35,
                      "LPE": 0.25, "DRIVEBY": 0.25, "SQLI": 0.20,
                      "PHISH": 0.15, "BRUTEFORCE": 0.15}.get(attack, 0.2)
            self.health = max(0.0, self.health - damage)
            # Unlock follow-on attacks
            for follow in ATTACK_CHAINS.get(attack, []):
                if follow not in self.unlocked:
                    self.unlocked.append(follow)
        self.history.append((attack, defense, breached))
        self.step += 1

    def history_str(self):
        if not self.history:
            return "No prior steps."
        lines = []
        for i, (atk, df, br) in enumerate(self.history):
            outcome = "BREACHED ⚠" if br else "BLOCKED ✓"
            lines.append(f"  Step {i+1}: Attack={atk} | Defense={df} | {outcome}")
        return "\n".join(lines)

class DuelEnv:
    def sample_scenario(self):
        return random.choice(SCENARIOS)

    def new_episode(self, sc):
        return EpisodeState(sc)

    def attacker_prompt(self, sc, ep: EpisodeState, defender_mem_line: str):
        # If prior breaches unlocked follow-on attacks, include them
        unlocked_str = ""
        if ep.unlocked:
            unlocked_str = f"Unlocked follow-on attacks (from prior breach): {', '.join(ep.unlocked)}\n"
        return (
            "[RED TEAM VS BLUE TEAM — MULTI-STEP DUEL]\n"
            f"Scenario: {sc['id']} | Step: {ep.step + 1}/{EPISODE_STEPS}\n"
            f"System health: {ep.health:.0%} ({ep.status()})\n"
            f"{sc['profile']}\n"
            f"{sc['hint']}\n"
            f"{ep.history_str()}\n"
            f"{unlocked_str}"
            f"{defender_mem_line}\n\n"
            "[ROLE] You are the Attacker (RED).\n"
            "Study the health, history, and profile — adapt your attack.\n"
            "Choose exactly one attack.\n"
            f"Valid attacks: {', '.join(ATTACKS)}\n\n"
            "Format exactly (one line):\n"
            "ATTACK: <" + "|".join(ATTACKS) + ">"
        )

    def defender_prompt(self, sc, ep: EpisodeState, attacker_move: str,
                        attacker_mem_lines: str):
        return (
            "[RED TEAM VS BLUE TEAM — MULTI-STEP DUEL]\n"
            f"Scenario: {sc['id']} | Step: {ep.step + 1}/{EPISODE_STEPS}\n"
            f"System health: {ep.health:.0%} ({ep.status()})\n"
            f"{sc['profile']}\n"
            f"{sc['hint']}\n"
            f"{ep.history_str()}\n"
            f"{attacker_mem_lines}\n"
            f"Attacker chose: {attacker_move}\n\n"
            "[ROLE] You are the Defender (BLUE).\n"
            "Read the health, history, and profile — adapt your defense.\n"
            "If health is DEGRADED/CRITICAL, prioritize recovery defenses.\n"
            "Choose exactly one defense.\n"
            f"Valid defenses: {', '.join(DEFENSES)}\n\n"
            "Format exactly (one line):\n"
            "DEFEND: <" + "|".join(DEFENSES) + ">"
        )

ENV = DuelEnv()

# AFTER — just set the constant, model loads inside Cell 5 loop
import torch
from unsloth import FastModel

MAX_SEQ_LENGTH = 512

🦥 Unsloth: Will patch your computer to enable 2x faster free finetuning.

2026-04-26 02:51:49.418031: E external/local_xla/xla/stream_executor/cuda/cuda_fft.cc:467] Unable to register cuFFT factory: Attempting to register factory for plugin cuFFT when one has already been registered
WARNING: All log messages before absl::InitializeLog() is called are written to STDERR
E0000 00:00:1777171909.909738      23 cuda_dnn.cc:8579] Unable to register cuDNN factory: Attempting to register factory for plugin cuDNN when one has already been registered
E0000 00:00:1777171910.055755      23 cuda_blas.cc:1407] Unable to register cuBLAS factory: Attempting to register factory for plugin cuBLAS when one has already been registered
W0000 00:00:1777171911.035422      23 computation_placer.cc:177] computation placer already registered. Please check linkage and avoid linking the same target more than once.
W0000 00:00:1777171911.035467      23 computation_placer.cc:177] computation placer already registered. Please check linkage and avoid linking the same target more than once.
W0000 00:00:1777171911.035470      23 computation_placer.cc:177] computation placer already registered. Please check linkage and avoid linking the same target more than once.
W0000 00:00:1777171911.035472      23 computation_placer.cc:177] computation placer already registered. Please check linkage and avoid linking the same target more than once.

INFO 04-26 02:52:35 [__init__.py:244] Automatically detected platform cuda.
ERROR 04-26 02:52:40 [fa_utils.py:57] Cannot use FA version 2 is not supported due to FA2 is only supported on devices with compute capability >= 8
🦥 Unsloth Zoo will now patch everything to make training faster!

from datasets import Dataset
from groq import Groq as GroqClient
import re
import json
import random
from pathlib import Path

OUTPUT_DIR = Path("outputs/grpo_training")
OUTPUT_DIR.mkdir(parents=True, exist_ok=True)

attack_re = re.compile(
    r"ATTACK:\s*(PHISH|BRUTEFORCE|DRIVEBY|RANSOM|SQLI|RCE|LPE|SUPPLYCHAIN)\b",
    re.IGNORECASE)
defend_re = re.compile(
    r"DEFEND:\s*(MFA|PATCH|EDR|BACKUP|WAF|LEASTPRIV|SBOM|ROTATEKEYS)\b",
    re.IGNORECASE)

def parse_attack(text: str):
    m = attack_re.search(text or "")
    return (m.group(1).upper() if m else "")

def parse_defense(text: str):
    m = defend_re.search(text or "")
    return (m.group(1).upper() if m else "")

def parse_scenario_and_attack_from_prompt(prompt: str):
    sc_id = ""
    m = re.search(r"Scenario:\s*(\w+)", prompt)
    if m:
        sc_id = m.group(1)
    atk = ""
    m2 = re.search(r"Attacker chose:\s*(\w+)", prompt)
    if m2:
        atk = m2.group(1).upper()
    sc = next((s for s in SCENARIOS if s["id"] == sc_id), None)
    return sc, atk

# AFTER
def outcome_breached(sc, attacker_move: str, defender_move: str,
                     ep: "EpisodeState" = None) -> bool:
    """Breached if:
    1. Attack hits the weakness AND defender chose wrong, OR
    2. Attack is a follow-on (unlocked) attack AND any defense fails to counter it
    """
    if not sc:
        return False
    # Check if attack is the scenario weakness or an unlocked follow-on
    is_weakness    = (attacker_move == sc["weakness"])
    is_followon    = (ep is not None) and (attacker_move in ep.unlocked)
    if not is_weakness and not is_followon:
        return False
    # For follow-on attacks use COUNTER_FALLBACK since they're not the scenario weakness
    if is_followon and not is_weakness:
        correct = COUNTER_FALLBACK.get(attacker_move, "PATCH")
    else:
        correct = get_counter(sc)
    return defender_move != correct

FORMAT_PENALTY = -1.0
# Partial credit proxy — used when defender picks wrong defense
# Maps (attack, defense) → approximate usefulness score
PARTIAL_CREDIT_PROXY = {
    ("PHISH",      "EDR"):        0.40,
    ("PHISH",      "PATCH"):      0.10,
    ("PHISH",      "BACKUP"):     0.05,
    ("BRUTEFORCE", "PATCH"):      0.18,
    ("BRUTEFORCE", "EDR"):        0.25,
    ("BRUTEFORCE", "WAF"):        0.30,
    ("DRIVEBY",    "EDR"):        0.62,
    ("DRIVEBY",    "MFA"):        0.05,
    ("RANSOM",     "EDR"):        0.55,
    ("RANSOM",     "PATCH"):      0.15,
    ("SQLI",       "PATCH"):      0.40,
    ("RCE",        "EDR"):        0.60,
    ("LPE",        "PATCH"):      0.45,
    ("SUPPLYCHAIN","PATCH"):      0.35,
}

def p_block(atk: str, df: str) -> float:
    return PARTIAL_CREDIT_PROXY.get((atk, df), 0.05)
# ── Groq LLM-as-Judge ────────────────────────────────────────────────────────
_groq = GroqClient(api_key=GROQ_API_KEY) if GROQ_API_KEY else None

JUDGE_SYSTEM = "You are a cybersecurity training evaluator. Return JSON only, no markdown."

DEFENDER_JUDGE_TMPL = """A defender chose a defense in a security scenario.

Scenario profile: {profile}
Log hint: {hint}
Attacker chose: {attack}
Defender chose: {defense}
Correct defense for this scenario: {correct}

Score the defender's choice considering the profile context.
Rules:
- score=1.0 only if defense matches the correct defense AND fits the profile
- score=0.3 to 0.6 if defense is partially useful given the profile
- score=-0.5 to -1.0 if defense ignores the profile entirely or is irrelevant

Return JSON only: {{"score": <-1.0 to 1.0>, "correct": <bool>, "reason": "<one sentence>"}}"""

ATTACKER_JUDGE_TMPL = """An attacker chose an attack in a security scenario.

Scenario profile: {profile}
Log hint: {hint}
Attacker chose: {attack}
Scenario weakness: {weakness}

Rules:
- score=1.0 if attack matches weakness AND is clearly justified by the profile/hint
- score=0.2 if attack matches weakness but ignores profile context
- score=-0.5 if attack is wrong but hint was genuinely ambiguous
- score=-1.0 if attack is completely wrong given obvious profile signals

Return JSON only: {{"score": <-1.0 to 1.0>, "correct": <bool>, "reason": "<one sentence>"}}"""

def _call_groq_judge(tmpl_str: str) -> float | None:
    """Call Groq judge, return float score or None on any failure."""
    if not _groq:
        return None
    try:
        resp = _groq.chat.completions.create(
            model=GROQ_MODEL,
            messages=[
                {"role": "system", "content": JUDGE_SYSTEM},
                {"role": "user",   "content": tmpl_str}
            ],
            temperature=0.1,
            max_tokens=120,
        )
        raw = resp.choices[0].message.content.strip()
        if raw.startswith("```"):
            raw = raw.split("\n", 1)[1].rsplit("```", 1)[0]
        data = json.loads(raw)
        return float(data.get("score", 0.0))
    except Exception:
        return None

def llm_judge_defender(sc, atk, df) -> float | None:
    return _call_groq_judge(DEFENDER_JUDGE_TMPL.format(
        profile=sc["profile"], hint=sc["hint"],
        attack=atk, defense=df, correct=get_counter(sc)))

def llm_judge_attacker(sc, atk) -> float | None:
    return _call_groq_judge(ATTACKER_JUDGE_TMPL.format(
        profile=sc["profile"], hint=sc["hint"],
        attack=atk, weakness=sc["weakness"]))
# REPLACE lines 411–492 entirely
def attacker_reward_func(prompts, completions, **kwargs):
    """Attacker reward: summed over episode steps with discount.
    Step 1 breach → +1.0. Follow-on breach (step 2-3) → +0.8 (slightly discounted).
    Unlocking a follow-on attack even without breach → +0.2 (partial credit).
    """
    rewards = []
    for prompt, completion in zip(prompts, completions):
        text = completion[0]["content"] if isinstance(completion, list) else completion
        atk  = parse_attack(text)
        sc, _ = parse_scenario_and_attack_from_prompt(prompt)

        if not atk or atk not in ATTACKS or not sc:
            rewards.append(FORMAT_PENALTY)
            continue

        # Parse episode state from prompt
        ep = _parse_episode_from_prompt(prompt)

        is_weakness  = (atk == sc["weakness"])
        is_followon  = (atk in ep.unlocked) if ep else False
        step_discount = 1.0 if (ep is None or ep.step == 0) else 0.8

        if is_weakness or is_followon:
            base_reward = 1.0 * step_discount
        elif ep and ep.unlocked:
            # Has unlocked attacks but chose something else — minor penalty
            base_reward = -0.3
        else:
            base_reward = -0.2

        if GROQ_API_KEY and random.random() < JUDGE_PROB:
            judge_score = llm_judge_attacker(sc, atk)
            reward = (0.7 * base_reward + 0.3 * judge_score
                      if judge_score is not None else base_reward)
        else:
            reward = base_reward

        rewards.append(reward)
    return rewards

def defender_reward_func(prompts, completions, **kwargs):
    """Defender reward: summed over episode steps with health bonus.
    Correct defense → +1.0.
    Correct defense when health DEGRADED/CRITICAL → +1.3 (recovery bonus).
    Wrong defense → partial credit via p_block proxy or -0.8.
    Health preservation bonus: +health * 0.3 (rewards keeping system alive).
    """
    rewards = []
    for prompt, completion in zip(prompts, completions):
        text = completion[0]["content"] if isinstance(completion, list) else completion
        df   = parse_defense(text)
        sc, atk = parse_scenario_and_attack_from_prompt(prompt)

        # AFTER
        if not df or df not in DEFENSES or not sc:
            rewards.append(FORMAT_PENALTY)
            continue
        if atk not in ATTACKS:
            atk = sc["weakness"] if sc else ""
        if not atk:
            rewards.append(FORMAT_PENALTY)
            continue

        ep      = _parse_episode_from_prompt(prompt)
        health  = ep.health if ep else 1.0
        status  = ep.status() if ep else "STABLE"
        correct = get_counter(sc)

        if df == correct:
            # Recovery bonus: harder to defend when already damaged
            recovery_bonus = 0.3 if status in ("DEGRADED", "CRITICAL") else 0.0
            base_reward    = 1.0 + recovery_bonus
        elif df in DEFENSES:
            try:
                partial     = p_block(atk, df) if atk in ATTACKS else 0.1
                base_reward = (partial * 2.0) - 1.0
                base_reward = max(base_reward, -0.8)
            except:
                base_reward = -0.5
        else:
            base_reward = -1.0

        # Health preservation bonus — rewards sustained good performance
        base_reward += health * 0.3

        if GROQ_API_KEY and random.random() < JUDGE_PROB:
            judge_score = llm_judge_defender(sc, atk, df)
            reward = (0.7 * base_reward + 0.3 * judge_score
                      if judge_score is not None else base_reward)
        else:
            reward = base_reward

        rewards.append(reward)
    return rewards

def _parse_episode_from_prompt(prompt: str) -> "EpisodeState":
    """Reconstruct a lightweight EpisodeState from the prompt text.
    Only needs health and unlocked attacks for reward computation."""
    sc_id = ""
    m = re.search(r"Scenario:\s*(\w+)", prompt)
    if m: sc_id = m.group(1)
    sc = next((s for s in SCENARIOS if s["id"] == sc_id), None)
    ep = EpisodeState(sc) if sc else EpisodeState(SCENARIOS[0])

    # Parse health from prompt
    mh = re.search(r"System health:\s*([\d.]+)%", prompt)
    if mh:
        ep.health = float(mh.group(1)) / 100.0

    # Parse prior step history from prompt to reconstruct unlocked attacks
    for line in prompt.split("\n"):
        m_breach = re.search(r"Attack=(\w+).*BREACHED", line)
        if m_breach:
            breached_atk = m_breach.group(1)
            for follow in ATTACK_CHAINS.get(breached_atk, []):
                if follow not in ep.unlocked:
                    ep.unlocked.append(follow)
    ep.step = len(ep.history)
    return ep


# ── Seed datasets (multi-step) ────────────────────────────────────────────────
def make_seed_datasets(n=SEED_EPISODES):
    """Generate seed prompts that sample from across all 3 episode steps.
    40% step-1 prompts (clean state), 35% step-2 prompts (some history),
    25% step-3 prompts (degraded state) — model sees full episode distribution."""
    atk_prompts, def_prompts = [], []
    def_mem_line  = attacker_memory_line()
    atk_mem_lines = defender_memory_lines()

    for _ in range(n):
        sc = ENV.sample_scenario()
        ep = ENV.new_episode(sc)

        # Simulate 0-2 prior steps to get varied episode states
        prior_steps = random.choices([0, 1, 2], weights=[0.40, 0.35, 0.25])[0]
        for _ in range(prior_steps):
            atk = sc["weakness"] if random.random() < 0.6 else random.choice(ATTACKS)
            df  = get_counter(sc) if random.random() < 0.4 else random.choice(DEFENSES)
            br  = outcome_breached(sc, atk, df, ep)
            ep.apply(atk, df, br)

        atk_prompts.append({"prompt": ENV.attacker_prompt(sc, ep, def_mem_line)})

    for _ in range(n):
        sc = ENV.sample_scenario()
        ep = ENV.new_episode(sc)

        prior_steps = random.choices([0, 1, 2], weights=[0.40, 0.35, 0.25])[0]
        for _ in range(prior_steps):
            atk = sc["weakness"] if random.random() < 0.6 else random.choice(ATTACKS)
            df  = get_counter(sc) if random.random() < 0.4 else random.choice(DEFENSES)
            br  = outcome_breached(sc, atk, df, ep)
            ep.apply(atk, df, br)

        # Always use weakness as attacker move for training signal clarity
        atk = sc["weakness"]
        def_prompts.append({"prompt": ENV.defender_prompt(sc, ep, atk, atk_mem_lines)})

    return Dataset.from_list(def_prompts), Dataset.from_list(atk_prompts)

train_dataset, attacker_dataset = make_seed_datasets(SEED_EPISODES)

import torch
from pathlib import Path
from transformers import TrainerCallback
from unsloth import FastModel
from trl import GRPOConfig, GRPOTrainer

class CLIProgressCallback(TrainerCallback):
    def __init__(self, tag: str, rolling=10):
        self.tag = tag
        self.rolling = rolling
        self._recent = []

    def on_log(self, args, state, control, logs=None, **kwargs):
        if not logs:
            return
        step   = state.global_step
        reward = logs.get("reward", logs.get("train/reward", None))
        if isinstance(reward, (int, float)):
            self._recent.append(float(reward))
            if len(self._recent) > self.rolling:
                self._recent = self._recent[-self.rolling:]
            mean_r = sum(self._recent) / len(self._recent)
            print(f"  [{self.tag}] step={step:3d} | reward={reward:+.4f} | "
                  f"mean(last{len(self._recent)}): {mean_r:+.4f}")
        elif reward is not None:
            print(f"  [{self.tag}] step={step:3d} | reward={reward}")

def _extract_rewards(log_history):
    steps, rewards = [], []
    for entry in log_history:
        s = entry.get("step")
        if not s:
            continue
        r = entry.get("reward", entry.get("train/reward", float("nan")))
        steps.append(s)
        rewards.append(r)
    return steps, rewards

def _make_lora(base_model_id):
    mdl, _ = FastModel.from_pretrained(
        model_name=base_model_id,
        max_seq_length=MAX_SEQ_LENGTH,
        load_in_4bit=True,
    )
    return FastModel.get_peft_model(
        mdl,
        r=16,
        target_modules=["q_proj","k_proj","v_proj","o_proj",
                        "gate_proj","up_proj","down_proj"],
        lora_alpha=32,
        lora_dropout=0,
        bias="none",
    )

def _grpo_config(output_dir, max_steps):
    return GRPOConfig(
        learning_rate=LEARNING_RATE,
        max_prompt_length=512,          # increased — multi-step prompts are longer
        max_completion_length=MAX_COMPLETION_LEN,
        num_generations=NUM_GENERATIONS,
        per_device_train_batch_size=BATCH_SIZE,
        gradient_accumulation_steps=GRAD_ACCUMULATION,
        max_steps=max_steps,
        beta=0.04,                      # KL penalty — prevents policy drift
        logging_steps=1,
        output_dir=str(output_dir),
    )

@torch.no_grad()
def sample_actions(model, tok, prompts, max_new_tokens=MAX_COMPLETION_LEN, temperature=0.4):
    FastModel.for_inference(model)
    outs = []
    for p in prompts:
        inputs = tok(
            p, return_tensors="pt", truncation=True, max_length=MAX_SEQ_LENGTH
        ).to(model.device)
        gen = model.generate(
            **inputs,
            max_new_tokens=max_new_tokens,
            temperature=temperature,
            do_sample=True,
            pad_token_id=tok.eos_token_id,
        )
        text = tok.decode(gen[0][inputs["input_ids"].shape[1]:], skip_special_tokens=True)
        outs.append(text)
    return outs

@torch.no_grad()
def rollout_duels(attacker_ckpt, defender_ckpt, n=30, verbose=3, update_memory=True):
    """Runs EPISODE_STEPS-step episodes. Tracks breach rate, health, and
    attack chain activations across each episode."""
    atk_m, atk_t = FastModel.from_pretrained(
        model_name=attacker_ckpt, max_seq_length=MAX_SEQ_LENGTH, load_in_4bit=True)
    df_m,  df_t  = FastModel.from_pretrained(
        model_name=defender_ckpt, max_seq_length=MAX_SEQ_LENGTH, load_in_4bit=True)

    total_breaches    = 0
    chain_activations = 0
    shown             = 0

    for i in range(n):
        sc = ENV.sample_scenario()
        ep = ENV.new_episode(sc)
        episode_breached = False

        if shown < verbose:
            print(f"\n--- Episode {i+1} | scenario={sc['id']} | weakness={sc['weakness']} ---")

        for step_i in range(EPISODE_STEPS):
            # ── Attacker move ──────────────────────────────────────────────
            atk_prompt = ENV.attacker_prompt(sc, ep, attacker_memory_line())
            atk_out    = sample_actions(atk_m, atk_t, [atk_prompt])[0]
            atk        = parse_attack(atk_out)
            if atk not in ATTACKS:
                atk = ""

            # ── Defender move ──────────────────────────────────────────────
            df_prompt = ENV.defender_prompt(
                sc, ep, atk or "UNKNOWN", defender_memory_lines())
            df_out    = sample_actions(df_m, df_t, [df_prompt])[0]
            df        = parse_defense(df_out)
            if df not in DEFENSES:
                df = ""

            # ── Outcome ────────────────────────────────────────────────────
            breached = False
            if atk in ATTACKS and df in DEFENSES:
                breached = outcome_breached(sc, atk, df, ep)
            elif atk in ATTACKS and df not in DEFENSES:
                # Invalid defender output → breach if attack hit weakness or follow-on
                breached = (atk == sc["weakness"]) or (atk in ep.unlocked)

            # Advance episode state (updates health, history, unlocked chains)
            ep.apply(atk, df, breached)

            if breached:
                episode_breached = True
            if ep.unlocked:
                chain_activations += 1

            # Update rolling memory
            if update_memory:
                if atk in ATTACKS:
                    recent_attacks.append(atk)
                if df in DEFENSES:
                    recent_defenses.append(df)
                if breached and atk in ATTACKS:
                    recent_breaches.append(atk)

            if shown < verbose:
                outcome_str = "BREACHED ⚠" if breached else "BLOCKED ✓"
                print(f"  Step {step_i+1}: Atk={atk or '(INVALID)':12s} | "
                      f"Def={df or '(INVALID)':12s} | {outcome_str} | "
                      f"Health={ep.health:.0%} | Chains={ep.unlocked}")

        total_breaches += int(episode_breached)
        if shown < verbose:
            print(f"  → Final health: {ep.health:.0%} ({ep.status()})")
            shown += 1

    atk_m.cpu(); df_m.cpu()
    del atk_m, df_m
    torch.cuda.empty_cache()

    return {
        "breach_rate":       total_breaches / max(1, n),
        "breaches":          total_breaches,
        "n":                 n,
        "chain_activations": chain_activations,
    }

# ── TRAIN LOOP ────────────────────────────────────────────────────────────────
training_results  = {}
def_out           = OUTPUT_DIR / "defender"
atk_out           = OUTPUT_DIR / "attacker"

all_def_steps,  all_def_rewards  = [], []
all_atk_steps,  all_atk_rewards  = [], []
step_offset_def, step_offset_atk = 0, 0

for phase in range(PHASE_ALTERNATIONS):
    print(f"\n── Phase {phase+1}/{PHASE_ALTERNATIONS} ──")

    # (A) Warm rolling memory with baseline multi-step duels
    print("Warming rolling memory (baseline multi-step duels)...")
    _ = rollout_duels(
        attacker_ckpt=TRAIN_MODEL_ID,
        defender_ckpt=TRAIN_MODEL_ID,
        n=20, verbose=0, update_memory=True)

    # ── 1) Train Attacker (GRPO) ───────────────────────────────────────────
    print(f"Training Attacker ({ATTACKER_STEPS} steps)...")
    atk_start = (str(atk_out)
                 if phase > 0 and (atk_out / "config.json").exists()
                 else TRAIN_MODEL_ID)
    atk_model   = _make_lora(atk_start)
    atk_trainer = GRPOTrainer(
        model=atk_model,
        reward_funcs=attacker_reward_func,
        args=_grpo_config(atk_out, ATTACKER_STEPS),
        train_dataset=attacker_dataset,
        callbacks=[CLIProgressCallback("ATTACK")],
    )
    atk_trainer.train()
    atk_trainer.save_model(str(atk_out))
    s, r = _extract_rewards(atk_trainer.state.log_history)
    all_atk_steps  += [x + step_offset_atk for x in s]
    all_atk_rewards += r
    step_offset_atk += ATTACKER_STEPS
    del atk_model, atk_trainer
    torch.cuda.empty_cache()

    # ── 2) Refresh both datasets with multi-step episode prompts ──────────
    print(f"Refreshing datasets ({REFRESH_EPISODES} episodes x {EPISODE_STEPS} steps)...")

    atk_inf, atk_tok = FastModel.from_pretrained(
        model_name=str(atk_out), max_seq_length=MAX_SEQ_LENGTH, load_in_4bit=True)

    refreshed_atk_prompts = []
    refreshed_def_prompts = []

    for _ in range(REFRESH_EPISODES):
        sc = ENV.sample_scenario()
        ep = ENV.new_episode(sc)

        for step_i in range(EPISODE_STEPS):
            # Build attacker prompt and sample its action
            atk_prompt = ENV.attacker_prompt(sc, ep, attacker_memory_line())
            atk_out_r  = sample_actions(atk_inf, atk_tok, [atk_prompt], temperature=0.4)[0]
            atk        = parse_attack(atk_out_r)
            if atk not in ATTACKS:
                atk = random.choice(ATTACKS)

            # Store this step's prompts for both agents
            refreshed_atk_prompts.append({"prompt": atk_prompt})
            refreshed_def_prompts.append({
                "prompt": ENV.defender_prompt(sc, ep, atk, defender_memory_lines())
            })

            # Simulate outcome to advance episode state for next step
            # Use correct defense ~50% of time so model sees both breach/block states
            df_sim   = (get_counter(sc)
                        if random.random() < 0.5
                        else random.choice(DEFENSES))
            breached = outcome_breached(sc, atk, df_sim, ep)
            ep.apply(atk, df_sim, breached)

            # Update rolling memory from simulated outcome
            if atk in ATTACKS:    recent_attacks.append(atk)
            if df_sim in DEFENSES: recent_defenses.append(df_sim)
            if breached:          recent_breaches.append(atk)

    del atk_inf
    torch.cuda.empty_cache()

    # Report refresh quality
    total_refresh = len(refreshed_atk_prompts)
    print(f"   ↳ Refreshed {total_refresh} attacker + {len(refreshed_def_prompts)} defender prompts")
    print(f"   ↳ Memory state: attacks={len(recent_attacks)} "
          f"defenses={len(recent_defenses)} breaches={len(recent_breaches)}")

    # Mix refreshed with existing dataset, cap at SEED_EPISODES
    attacker_dataset = Dataset.from_list(
        (refreshed_atk_prompts
         + [{"prompt": p["prompt"]} for p in attacker_dataset.select(
             range(min(len(attacker_dataset),
                       SEED_EPISODES - len(refreshed_atk_prompts))))]
         )[:SEED_EPISODES])

    train_dataset = Dataset.from_list(
        (refreshed_def_prompts
         + [{"prompt": p["prompt"]} for p in train_dataset.select(
             range(min(len(train_dataset),
                       SEED_EPISODES - len(refreshed_def_prompts))))]
         )[:SEED_EPISODES])

    print(f"Datasets: attacker={len(attacker_dataset)} defender={len(train_dataset)}")

    # ── 3) Train Defender (GRPO) ───────────────────────────────────────────
    print(f"Training Defender ({MAX_STEPS} steps)...")
    def_start = (str(def_out)
                 if phase > 0 and (def_out / "config.json").exists()
                 else TRAIN_MODEL_ID)
    def_model   = _make_lora(def_start)
    def_trainer = GRPOTrainer(
        model=def_model,
        reward_funcs=defender_reward_func,
        args=_grpo_config(def_out, MAX_STEPS),
        train_dataset=train_dataset,
        callbacks=[CLIProgressCallback("DEFEND")],
    )
    def_trainer.train()
    def_trainer.save_model(str(def_out))
    s, r = _extract_rewards(def_trainer.state.log_history)
    all_def_steps  += [x + step_offset_def for x in s]
    all_def_rewards += r
    step_offset_def += MAX_STEPS
    del def_model, def_trainer
    torch.cuda.empty_cache()

    # ── 4) Sanity rollout — multi-step, shows attack chains in action ─────
    print("\nQuick rollout (trained attacker vs trained defender):")
    stats = rollout_duels(
        attacker_ckpt=str(atk_out),
        defender_ckpt=str(def_out),
        n=25, verbose=3, update_memory=True)
    print(f"Breach rate: {stats['breach_rate']:.2%} "
          f"({stats['breaches']}/{stats['n']}) | "
          f"Chain activations: {stats['chain_activations']}")

# ── Save results ──────────────────────────────────────────────────────────────
training_results["standard"] = {
    "steps":      all_def_steps,
    "rewards":    all_def_rewards,
    "atk_steps":  all_atk_steps,
    "atk_rewards": all_atk_rewards,
    "output_dir": def_out,
    "atk_dir":    atk_out,
}

# Save tokenizers alongside checkpoints for eval/reuse
try:
    _, base_tok = FastModel.from_pretrained(
        model_name=TRAIN_MODEL_ID, max_seq_length=MAX_SEQ_LENGTH, load_in_4bit=False)
    base_tok.save_pretrained(str(atk_out))
    base_tok.save_pretrained(str(def_out))
    print("Tokenizer saved to attacker/defender output dirs")
except Exception as e:
    print("Could not save tokenizer:", e)

print("Training complete")
print(f"Attacker checkpoint: {atk_out}")
print(f"Defender checkpoint: {def_out}")

── Phase 1/1 ──
Warming rolling memory (baseline multi-step duels)...
==((====))==  Unsloth 2026.4.8: Fast Qwen2 patching. Transformers: 4.56.2. vLLM: 0.9.2.
   \\   /|    Tesla T4. Num GPUs = 2. Max memory: 14.563 GB. Platform: Linux.
O^O/ \_/ \    Torch: 2.7.0+cu126. CUDA: 7.5. CUDA Toolkit: 12.6. Triton: 3.2.0
\        /    Bfloat16 = FALSE. FA [Xformers = 0.0.30. FA2 = False]
 "-____-"     Free license: http://github.com/unslothai/unsloth
Unsloth: Fast downloading is enabled - ignore downloading bars which are red colored!

unsloth/qwen2.5-0.5b-instruct-unsloth-bnb-4bit does not have a padding token! Will use pad_token = <|PAD_TOKEN|>.
==((====))==  Unsloth 2026.4.8: Fast Qwen2 patching. Transformers: 4.56.2. vLLM: 0.9.2.
   \\   /|    Tesla T4. Num GPUs = 2. Max memory: 14.563 GB. Platform: Linux.
O^O/ \_/ \    Torch: 2.7.0+cu126. CUDA: 7.5. CUDA Toolkit: 12.6. Triton: 3.2.0
\        /    Bfloat16 = FALSE. FA [Xformers = 0.0.30. FA2 = False]
 "-____-"     Free license: http://github.com/unslothai/unsloth
Unsloth: Fast downloading is enabled - ignore downloading bars which are red colored!
unsloth/qwen2.5-0.5b-instruct-unsloth-bnb-4bit does not have a padding token! Will use pad_token = <|PAD_TOKEN|>.
Training Attacker (400 steps)...
==((====))==  Unsloth 2026.4.8: Fast Qwen2 patching. Transformers: 4.56.2. vLLM: 0.9.2.
   \\   /|    Tesla T4. Num GPUs = 2. Max memory: 14.563 GB. Platform: Linux.
O^O/ \_/ \    Torch: 2.7.0+cu126. CUDA: 7.5. CUDA Toolkit: 12.6. Triton: 3.2.0
\        /    Bfloat16 = FALSE. FA [Xformers = 0.0.30. FA2 = False]
 "-____-"     Free license: http://github.com/unslothai/unsloth
Unsloth: Fast downloading is enabled - ignore downloading bars which are red colored!
unsloth/qwen2.5-0.5b-instruct-unsloth-bnb-4bit does not have a padding token! Will use pad_token = <|PAD_TOKEN|>.
Unsloth: We now expect `per_device_train_batch_size` * `gradient_accumulation_steps` * `world_size` to be a multiple of `num_generations`.
We will change the batch size of 1 to the `num_generations` of 4

==((====))==  Unsloth - 2x faster free finetuning | Num GPUs used = 1
   \\   /|    Num examples = 256 | Num Epochs = 4 | Total steps = 400
O^O/ \_/ \    Batch size per device = 4 | Gradient accumulation steps = 2
\        /    Data Parallel GPUs = 1 | Total batch size (4 x 2 x 1) = 8
 "-____-"     Trainable parameters = 8,798,208 of 502,830,976 (1.75% trained)

Unsloth: Will smartly offload gradients to save VRAM!

  [ATTACK] step=  1 | reward=-0.7500 | mean(last1): -0.7500
  [ATTACK] step=  2 | reward=-0.9000 | mean(last2): -0.8250
  [ATTACK] step=  3 | reward=-0.7500 | mean(last3): -0.8000
  [ATTACK] step=  4 | reward=-0.9000 | mean(last4): -0.8250
  [ATTACK] step=  5 | reward=-1.0000 | mean(last5): -0.8600
  [ATTACK] step=  6 | reward=-1.0000 | mean(last6): -0.8833
  [ATTACK] step=  7 | reward=-0.9000 | mean(last7): -0.8857
  [ATTACK] step=  8 | reward=-0.6000 | mean(last8): -0.8500
  [ATTACK] step=  9 | reward=-0.7500 | mean(last9): -0.8389
  [ATTACK] step= 10 | reward=-0.6500 | mean(last10): -0.8200
  [ATTACK] step= 11 | reward=-1.0000 | mean(last10): -0.8450
  [ATTACK] step= 12 | reward=-1.0000 | mean(last10): -0.8550
  [ATTACK] step= 13 | reward=-0.9125 | mean(last10): -0.8712
  [ATTACK] step= 14 | reward=-0.9000 | mean(last10): -0.8712
  [ATTACK] step= 15 | reward=-1.0000 | mean(last10): -0.8712
  [ATTACK] step= 16 | reward=-0.7500 | mean(last10): -0.8462
  [ATTACK] step= 17 | reward=-0.4000 | mean(last10): -0.7963
  [ATTACK] step= 18 | reward=-0.7500 | mean(last10): -0.8112
  [ATTACK] step= 19 | reward=-0.5000 | mean(last10): -0.7862
  [ATTACK] step= 20 | reward=-0.2500 | mean(last10): -0.7463
  [ATTACK] step= 21 | reward=-0.5500 | mean(last10): -0.7013
  [ATTACK] step= 22 | reward=-0.7500 | mean(last10): -0.6763
  [ATTACK] step= 23 | reward=-1.0000 | mean(last10): -0.6850
  [ATTACK] step= 24 | reward=-0.5000 | mean(last10): -0.6450
  [ATTACK] step= 25 | reward=-0.7500 | mean(last10): -0.6200
  [ATTACK] step= 26 | reward=-1.0000 | mean(last10): -0.6450
  [ATTACK] step= 27 | reward=-0.7500 | mean(last10): -0.6800
  [ATTACK] step= 28 | reward=-0.6625 | mean(last10): -0.6713
  [ATTACK] step= 29 | reward=-0.7500 | mean(last10): -0.6963
  [ATTACK] step= 30 | reward=-0.6500 | mean(last10): -0.7363
  [ATTACK] step= 31 | reward=-0.1500 | mean(last10): -0.6963
  [ATTACK] step= 32 | reward=-1.0000 | mean(last10): -0.7213
  [ATTACK] step= 33 | reward=-1.0000 | mean(last10): -0.7213
  [ATTACK] step= 34 | reward=-1.0000 | mean(last10): -0.7713
  [ATTACK] step= 35 | reward=-0.7500 | mean(last10): -0.7713
  [ATTACK] step= 36 | reward=-0.9000 | mean(last10): -0.7612
  [ATTACK] step= 37 | reward=-0.4500 | mean(last10): -0.7312
  [ATTACK] step= 38 | reward=-0.7500 | mean(last10): -0.7400
  [ATTACK] step= 39 | reward=-0.7500 | mean(last10): -0.7400
  [ATTACK] step= 40 | reward=-0.7500 | mean(last10): -0.7500
  [ATTACK] step= 41 | reward=-0.7500 | mean(last10): -0.8100
  [ATTACK] step= 42 | reward=-0.7500 | mean(last10): -0.7850
  [ATTACK] step= 43 | reward=-0.9125 | mean(last10): -0.7763
  [ATTACK] step= 44 | reward=-0.7500 | mean(last10): -0.7513
  [ATTACK] step= 45 | reward=-0.9000 | mean(last10): -0.7662
  [ATTACK] step= 46 | reward=-0.3000 | mean(last10): -0.7063
  [ATTACK] step= 47 | reward=-0.7500 | mean(last10): -0.7363
  [ATTACK] step= 48 | reward=-1.0000 | mean(last10): -0.7613
  [ATTACK] step= 49 | reward=-0.8000 | mean(last10): -0.7663
  [ATTACK] step= 50 | reward=-1.0000 | mean(last10): -0.7913
  [ATTACK] step= 51 | reward=-0.9000 | mean(last10): -0.8063
  [ATTACK] step= 52 | reward=-0.8000 | mean(last10): -0.8113
  [ATTACK] step= 53 | reward=-0.5000 | mean(last10): -0.7700
  [ATTACK] step= 54 | reward=-0.7500 | mean(last10): -0.7700
  [ATTACK] step= 55 | reward=-0.9125 | mean(last10): -0.7713
  [ATTACK] step= 56 | reward=-0.7500 | mean(last10): -0.8163
  [ATTACK] step= 57 | reward=-0.6500 | mean(last10): -0.8063
  [ATTACK] step= 58 | reward=-0.1500 | mean(last10): -0.7213
  [ATTACK] step= 59 | reward=-0.5500 | mean(last10): -0.6963
  [ATTACK] step= 60 | reward=-0.4000 | mean(last10): -0.6363
  [ATTACK] step= 61 | reward=-0.6625 | mean(last10): -0.6125
  [ATTACK] step= 62 | reward=-0.9125 | mean(last10): -0.6238
  [ATTACK] step= 63 | reward=-0.5000 | mean(last10): -0.6238
  [ATTACK] step= 64 | reward=-0.7500 | mean(last10): -0.6238
  [ATTACK] step= 65 | reward=-0.6500 | mean(last10): -0.5975
  [ATTACK] step= 66 | reward=-0.5000 | mean(last10): -0.5725
  [ATTACK] step= 67 | reward=-0.6500 | mean(last10): -0.5725
  [ATTACK] step= 68 | reward=-0.4000 | mean(last10): -0.5975
  [ATTACK] step= 69 | reward=-0.7500 | mean(last10): -0.6175
  [ATTACK] step= 70 | reward=-1.0000 | mean(last10): -0.6775
  [ATTACK] step= 71 | reward=-0.7500 | mean(last10): -0.6862
  [ATTACK] step= 72 | reward=+0.0000 | mean(last10): -0.5950
  [ATTACK] step= 73 | reward=-0.9000 | mean(last10): -0.6350
  [ATTACK] step= 74 | reward=-0.3000 | mean(last10): -0.5900
  [ATTACK] step= 75 | reward=-0.5000 | mean(last10): -0.5750
  [ATTACK] step= 76 | reward=-0.6500 | mean(last10): -0.5900
  [ATTACK] step= 77 | reward=-0.6000 | mean(last10): -0.5850
  [ATTACK] step= 78 | reward=-0.4000 | mean(last10): -0.5850
  [ATTACK] step= 79 | reward=-0.8000 | mean(last10): -0.5900
  [ATTACK] step= 80 | reward=-0.6500 | mean(last10): -0.5550
  [ATTACK] step= 81 | reward=-0.4000 | mean(last10): -0.5200
  [ATTACK] step= 82 | reward=-0.4500 | mean(last10): -0.5650
  [ATTACK] step= 83 | reward=-0.9000 | mean(last10): -0.5650
  [ATTACK] step= 84 | reward=-0.9000 | mean(last10): -0.6250
  [ATTACK] step= 85 | reward=-0.5000 | mean(last10): -0.6250
  [ATTACK] step= 86 | reward=-0.7500 | mean(last10): -0.6350
  [ATTACK] step= 87 | reward=-0.6000 | mean(last10): -0.6350
  [ATTACK] step= 88 | reward=-0.6500 | mean(last10): -0.6600
  [ATTACK] step= 89 | reward=-0.9125 | mean(last10): -0.6712
  [ATTACK] step= 90 | reward=-0.3000 | mean(last10): -0.6363
  [ATTACK] step= 91 | reward=-0.2250 | mean(last10): -0.6188
  [ATTACK] step= 92 | reward=-0.7500 | mean(last10): -0.6487
  [ATTACK] step= 93 | reward=-0.9000 | mean(last10): -0.6487
  [ATTACK] step= 94 | reward=-0.6000 | mean(last10): -0.6188
  [ATTACK] step= 95 | reward=-0.5000 | mean(last10): -0.6188
  [ATTACK] step= 96 | reward=-0.5000 | mean(last10): -0.5938
  [ATTACK] step= 97 | reward=-0.5000 | mean(last10): -0.5838
  [ATTACK] step= 98 | reward=-0.8250 | mean(last10): -0.6013
  [ATTACK] step= 99 | reward=-0.6625 | mean(last10): -0.5763
  [ATTACK] step=100 | reward=-0.3000 | mean(last10): -0.5763
  [ATTACK] step=101 | reward=-0.5000 | mean(last10): -0.6038
  [ATTACK] step=102 | reward=-0.6500 | mean(last10): -0.5938
  [ATTACK] step=103 | reward=-0.9000 | mean(last10): -0.5938
  [ATTACK] step=104 | reward=-0.5500 | mean(last10): -0.5887
  [ATTACK] step=105 | reward=-0.9000 | mean(last10): -0.6287
  [ATTACK] step=106 | reward=-0.2000 | mean(last10): -0.5987
  [ATTACK] step=107 | reward=-0.9125 | mean(last10): -0.6400
  [ATTACK] step=108 | reward=-1.0000 | mean(last10): -0.6575
  [ATTACK] step=109 | reward=-0.8000 | mean(last10): -0.6713
  [ATTACK] step=110 | reward=-0.6500 | mean(last10): -0.7062
  [ATTACK] step=111 | reward=-1.0000 | mean(last10): -0.7562
  [ATTACK] step=112 | reward=-0.4000 | mean(last10): -0.7312
  [ATTACK] step=113 | reward=-0.5500 | mean(last10): -0.6963
  [ATTACK] step=114 | reward=-0.4000 | mean(last10): -0.6813
  [ATTACK] step=115 | reward=-0.9000 | mean(last10): -0.6813
  [ATTACK] step=116 | reward=-0.1000 | mean(last10): -0.6713
  [ATTACK] step=117 | reward=-0.5500 | mean(last10): -0.6350
  [ATTACK] step=118 | reward=-0.6500 | mean(last10): -0.6000
  [ATTACK] step=119 | reward=-0.6500 | mean(last10): -0.5850
  [ATTACK] step=120 | reward=-0.6000 | mean(last10): -0.5800
  [ATTACK] step=121 | reward=-0.5500 | mean(last10): -0.5350
  [ATTACK] step=122 | reward=-0.1500 | mean(last10): -0.5100
  [ATTACK] step=123 | reward=-0.9000 | mean(last10): -0.5450
  [ATTACK] step=124 | reward=-0.5000 | mean(last10): -0.5550
  [ATTACK] step=125 | reward=-0.1500 | mean(last10): -0.4800
  [ATTACK] step=126 | reward=-0.4000 | mean(last10): -0.5100
  [ATTACK] step=127 | reward=-0.6000 | mean(last10): -0.5150
  [ATTACK] step=128 | reward=+0.0000 | mean(last10): -0.4500
  [ATTACK] step=129 | reward=-0.6500 | mean(last10): -0.4500
  [ATTACK] step=130 | reward=-0.3000 | mean(last10): -0.4200
  [ATTACK] step=131 | reward=-0.8000 | mean(last10): -0.4450
  [ATTACK] step=132 | reward=-0.5000 | mean(last10): -0.4800
  [ATTACK] step=133 | reward=-0.7125 | mean(last10): -0.4613
  [ATTACK] step=134 | reward=-0.0500 | mean(last10): -0.4163
  [ATTACK] step=135 | reward=-0.5000 | mean(last10): -0.4513
  [ATTACK] step=136 | reward=-0.9000 | mean(last10): -0.5012
  [ATTACK] step=137 | reward=-0.3000 | mean(last10): -0.4712
  [ATTACK] step=138 | reward=-0.1625 | mean(last10): -0.4875
  [ATTACK] step=139 | reward=-0.4000 | mean(last10): -0.4625
  [ATTACK] step=140 | reward=-0.7125 | mean(last10): -0.5037
  [ATTACK] step=141 | reward=-0.4000 | mean(last10): -0.4637
  [ATTACK] step=142 | reward=-0.3000 | mean(last10): -0.4437
  [ATTACK] step=143 | reward=-0.5500 | mean(last10): -0.4275
  [ATTACK] step=144 | reward=-0.3125 | mean(last10): -0.4537
  [ATTACK] step=145 | reward=-0.9000 | mean(last10): -0.4937
  [ATTACK] step=146 | reward=-0.5000 | mean(last10): -0.4537
  [ATTACK] step=147 | reward=-0.4000 | mean(last10): -0.4637
  [ATTACK] step=148 | reward=-0.5625 | mean(last10): -0.5037
  [ATTACK] step=149 | reward=-0.7000 | mean(last10): -0.5338
  [ATTACK] step=150 | reward=-0.6500 | mean(last10): -0.5275
  [ATTACK] step=151 | reward=-0.4000 | mean(last10): -0.5275
  [ATTACK] step=152 | reward=-0.6000 | mean(last10): -0.5575
  [ATTACK] step=153 | reward=-0.4500 | mean(last10): -0.5475
  [ATTACK] step=154 | reward=-0.2000 | mean(last10): -0.5363
  [ATTACK] step=155 | reward=+0.2500 | mean(last10): -0.4213
  [ATTACK] step=156 | reward=-0.3500 | mean(last10): -0.4063
  [ATTACK] step=157 | reward=-0.3250 | mean(last10): -0.3988
  [ATTACK] step=158 | reward=+0.1000 | mean(last10): -0.3325
  [ATTACK] step=159 | reward=-0.1000 | mean(last10): -0.2725
  [ATTACK] step=160 | reward=-0.2000 | mean(last10): -0.2275
  [ATTACK] step=161 | reward=+0.1000 | mean(last10): -0.1775
  [ATTACK] step=162 | reward=-0.5750 | mean(last10): -0.1750
  [ATTACK] step=163 | reward=-0.4000 | mean(last10): -0.1700
  [ATTACK] step=164 | reward=-0.7125 | mean(last10): -0.2213
  [ATTACK] step=165 | reward=-0.6625 | mean(last10): -0.3125
  [ATTACK] step=166 | reward=-0.5750 | mean(last10): -0.3350
  [ATTACK] step=167 | reward=+0.1000 | mean(last10): -0.2925
  [ATTACK] step=168 | reward=-0.4500 | mean(last10): -0.3475
  [ATTACK] step=169 | reward=-0.3000 | mean(last10): -0.3675
  [ATTACK] step=170 | reward=-0.5625 | mean(last10): -0.4038
  [ATTACK] step=171 | reward=-0.4000 | mean(last10): -0.4538
  [ATTACK] step=172 | reward=-0.4000 | mean(last10): -0.4363
  [ATTACK] step=173 | reward=-0.5500 | mean(last10): -0.4513
  [ATTACK] step=174 | reward=-0.3250 | mean(last10): -0.4125
  [ATTACK] step=175 | reward=-0.1000 | mean(last10): -0.3563
  [ATTACK] step=176 | reward=-0.5500 | mean(last10): -0.3538
  [ATTACK] step=177 | reward=-0.2000 | mean(last10): -0.3838
  [ATTACK] step=178 | reward=-0.5000 | mean(last10): -0.3888
  [ATTACK] step=179 | reward=+0.2500 | mean(last10): -0.3338
  [ATTACK] step=180 | reward=-0.6625 | mean(last10): -0.3438
  [ATTACK] step=181 | reward=-0.7000 | mean(last10): -0.3738
  [ATTACK] step=182 | reward=-0.4500 | mean(last10): -0.3788
  [ATTACK] step=183 | reward=-0.4500 | mean(last10): -0.3688
  [ATTACK] step=184 | reward=-0.0500 | mean(last10): -0.3413
  [ATTACK] step=185 | reward=-0.1500 | mean(last10): -0.3463
  [ATTACK] step=186 | reward=-0.3000 | mean(last10): -0.3213
  [ATTACK] step=187 | reward=-0.3000 | mean(last10): -0.3313
  [ATTACK] step=188 | reward=-0.7500 | mean(last10): -0.3563
  [ATTACK] step=189 | reward=-0.4500 | mean(last10): -0.4263
  [ATTACK] step=190 | reward=-0.5000 | mean(last10): -0.4100
  [ATTACK] step=191 | reward=-0.5000 | mean(last10): -0.3900
  [ATTACK] step=192 | reward=-0.3500 | mean(last10): -0.3800
  [ATTACK] step=193 | reward=-0.5000 | mean(last10): -0.3850
  [ATTACK] step=194 | reward=-0.1500 | mean(last10): -0.3950
  [ATTACK] step=195 | reward=-0.5500 | mean(last10): -0.4350
  [ATTACK] step=196 | reward=+0.0500 | mean(last10): -0.4000
  [ATTACK] step=197 | reward=+0.2500 | mean(last10): -0.3450
  [ATTACK] step=198 | reward=-0.4000 | mean(last10): -0.3100
  [ATTACK] step=199 | reward=-0.5500 | mean(last10): -0.3200
  [ATTACK] step=200 | reward=-0.5500 | mean(last10): -0.3250
  [ATTACK] step=201 | reward=-0.5500 | mean(last10): -0.3300
  [ATTACK] step=202 | reward=-0.7000 | mean(last10): -0.3650
  [ATTACK] step=203 | reward=-0.2000 | mean(last10): -0.3350
  [ATTACK] step=204 | reward=-0.1500 | mean(last10): -0.3350
  [ATTACK] step=205 | reward=-0.4500 | mean(last10): -0.3250
  [ATTACK] step=206 | reward=-0.1500 | mean(last10): -0.3450
  [ATTACK] step=207 | reward=-0.0625 | mean(last10): -0.3763
  [ATTACK] step=208 | reward=+0.0000 | mean(last10): -0.3363
  [ATTACK] step=209 | reward=-0.2500 | mean(last10): -0.3063
  [ATTACK] step=210 | reward=+0.2000 | mean(last10): -0.2313
  [ATTACK] step=211 | reward=+0.1000 | mean(last10): -0.1663
  [ATTACK] step=212 | reward=+0.2000 | mean(last10): -0.0763
  [ATTACK] step=213 | reward=-0.0500 | mean(last10): -0.0613
  [ATTACK] step=214 | reward=-0.7500 | mean(last10): -0.1213
  [ATTACK] step=215 | reward=-0.7000 | mean(last10): -0.1463
  [ATTACK] step=216 | reward=+0.0500 | mean(last10): -0.1263
  [ATTACK] step=217 | reward=-0.5500 | mean(last10): -0.1750
  [ATTACK] step=218 | reward=-0.4000 | mean(last10): -0.2150
  [ATTACK] step=219 | reward=+0.3000 | mean(last10): -0.1600
  [ATTACK] step=220 | reward=-0.4500 | mean(last10): -0.2250
  [ATTACK] step=221 | reward=+0.0500 | mean(last10): -0.2300
  [ATTACK] step=222 | reward=-0.5500 | mean(last10): -0.3050
  [ATTACK] step=223 | reward=-0.2125 | mean(last10): -0.3213
  [ATTACK] step=224 | reward=-0.1500 | mean(last10): -0.2613
  [ATTACK] step=225 | reward=-0.2125 | mean(last10): -0.2125
  [ATTACK] step=226 | reward=-0.3125 | mean(last10): -0.2488
  [ATTACK] step=227 | reward=-0.7000 | mean(last10): -0.2638
  [ATTACK] step=228 | reward=-0.2500 | mean(last10): -0.2488
  [ATTACK] step=229 | reward=-0.2125 | mean(last10): -0.3000
  [ATTACK] step=230 | reward=+0.1875 | mean(last10): -0.2363
  [ATTACK] step=231 | reward=+0.7000 | mean(last10): -0.1713
  [ATTACK] step=232 | reward=+0.3000 | mean(last10): -0.0863
  [ATTACK] step=233 | reward=-0.2500 | mean(last10): -0.0900
  [ATTACK] step=234 | reward=-0.4875 | mean(last10): -0.1238
  [ATTACK] step=235 | reward=-0.4000 | mean(last10): -0.1425
  [ATTACK] step=236 | reward=-0.3500 | mean(last10): -0.1463
  [ATTACK] step=237 | reward=-0.4500 | mean(last10): -0.1213
  [ATTACK] step=238 | reward=-0.4500 | mean(last10): -0.1413
  [ATTACK] step=239 | reward=-0.4000 | mean(last10): -0.1600
  [ATTACK] step=240 | reward=-0.1750 | mean(last10): -0.1963
  [ATTACK] step=241 | reward=+0.4500 | mean(last10): -0.2213
  [ATTACK] step=242 | reward=-0.2000 | mean(last10): -0.2713
  [ATTACK] step=243 | reward=-0.1000 | mean(last10): -0.2563
  [ATTACK] step=244 | reward=-0.3500 | mean(last10): -0.2425
  [ATTACK] step=245 | reward=+0.4500 | mean(last10): -0.1575
  [ATTACK] step=246 | reward=-0.3500 | mean(last10): -0.1575
  [ATTACK] step=247 | reward=-0.6125 | mean(last10): -0.1738
  [ATTACK] step=248 | reward=-0.4500 | mean(last10): -0.1738
  [ATTACK] step=249 | reward=-0.4625 | mean(last10): -0.1800
  [ATTACK] step=250 | reward=-0.4000 | mean(last10): -0.2025
  [ATTACK] step=251 | reward=-0.7125 | mean(last10): -0.3188
  [ATTACK] step=252 | reward=-0.3125 | mean(last10): -0.3300
  [ATTACK] step=253 | reward=-0.2500 | mean(last10): -0.3450
  [ATTACK] step=254 | reward=-0.0750 | mean(last10): -0.3175
  [ATTACK] step=255 | reward=-0.2125 | mean(last10): -0.3838
  [ATTACK] step=256 | reward=+0.0500 | mean(last10): -0.3438
  [ATTACK] step=257 | reward=-0.8000 | mean(last10): -0.3625
  [ATTACK] step=258 | reward=+0.1500 | mean(last10): -0.3025
  [ATTACK] step=259 | reward=-0.3250 | mean(last10): -0.2887
  [ATTACK] step=260 | reward=-0.0625 | mean(last10): -0.2550
  [ATTACK] step=261 | reward=-0.0500 | mean(last10): -0.1888
  [ATTACK] step=262 | reward=-0.0750 | mean(last10): -0.1650
  [ATTACK] step=263 | reward=+0.1000 | mean(last10): -0.1300
  [ATTACK] step=264 | reward=-0.3625 | mean(last10): -0.1588
  [ATTACK] step=265 | reward=-0.6000 | mean(last10): -0.1975
  [ATTACK] step=266 | reward=-0.2500 | mean(last10): -0.2275
  [ATTACK] step=267 | reward=-0.2875 | mean(last10): -0.1763
  [ATTACK] step=268 | reward=-0.3000 | mean(last10): -0.2213
  [ATTACK] step=269 | reward=+0.3500 | mean(last10): -0.1538
  [ATTACK] step=270 | reward=-0.2500 | mean(last10): -0.1725
  [ATTACK] step=271 | reward=-0.4500 | mean(last10): -0.2125
  [ATTACK] step=272 | reward=-0.2000 | mean(last10): -0.2250
  [ATTACK] step=273 | reward=+0.2000 | mean(last10): -0.2150
  [ATTACK] step=274 | reward=-0.0500 | mean(last10): -0.1838
  [ATTACK] step=275 | reward=-0.1500 | mean(last10): -0.1388
  [ATTACK] step=276 | reward=-0.9000 | mean(last10): -0.2038
  [ATTACK] step=277 | reward=-0.0500 | mean(last10): -0.1800
  [ATTACK] step=278 | reward=+0.3000 | mean(last10): -0.1200
  [ATTACK] step=279 | reward=+0.5250 | mean(last10): -0.1025
  [ATTACK] step=280 | reward=+0.1750 | mean(last10): -0.0600
  [ATTACK] step=281 | reward=-0.3000 | mean(last10): -0.0450
  [ATTACK] step=282 | reward=-0.4875 | mean(last10): -0.0738
  [ATTACK] step=283 | reward=+0.3000 | mean(last10): -0.0638
  [ATTACK] step=284 | reward=-0.1500 | mean(last10): -0.0738
  [ATTACK] step=285 | reward=-0.4625 | mean(last10): -0.1050
  [ATTACK] step=286 | reward=+0.3500 | mean(last10): +0.0200
  [ATTACK] step=287 | reward=+0.0000 | mean(last10): +0.0250
  [ATTACK] step=288 | reward=+0.2000 | mean(last10): +0.0150
  [ATTACK] step=289 | reward=-0.2250 | mean(last10): -0.0600
  [ATTACK] step=290 | reward=-0.4500 | mean(last10): -0.1225
  [ATTACK] step=291 | reward=-0.2500 | mean(last10): -0.1175
  [ATTACK] step=292 | reward=-0.0250 | mean(last10): -0.0713
  [ATTACK] step=293 | reward=+0.5000 | mean(last10): -0.0513
  [ATTACK] step=294 | reward=+0.0500 | mean(last10): -0.0313
  [ATTACK] step=295 | reward=+0.0500 | mean(last10): +0.0200
  [ATTACK] step=296 | reward=+0.1875 | mean(last10): +0.0038
  [ATTACK] step=297 | reward=+0.1000 | mean(last10): +0.0137
  [ATTACK] step=298 | reward=-0.6000 | mean(last10): -0.0663
  [ATTACK] step=299 | reward=+0.3000 | mean(last10): -0.0138
  [ATTACK] step=300 | reward=+0.6000 | mean(last10): +0.0913
  [ATTACK] step=301 | reward=-0.3000 | mean(last10): +0.0862
  [ATTACK] step=302 | reward=-0.3000 | mean(last10): +0.0587
  [ATTACK] step=303 | reward=+0.0500 | mean(last10): +0.0137
  [ATTACK] step=304 | reward=+0.3500 | mean(last10): +0.0437
  [ATTACK] step=305 | reward=-0.5000 | mean(last10): -0.0113
  [ATTACK] step=306 | reward=-0.3500 | mean(last10): -0.0650
  [ATTACK] step=307 | reward=-0.3000 | mean(last10): -0.1050
  [ATTACK] step=308 | reward=-0.1500 | mean(last10): -0.0600
  [ATTACK] step=309 | reward=-0.4750 | mean(last10): -0.1375
  [ATTACK] step=310 | reward=-0.2500 | mean(last10): -0.2225
  [ATTACK] step=311 | reward=-0.2000 | mean(last10): -0.2125
  [ATTACK] step=312 | reward=-0.4750 | mean(last10): -0.2300
  [ATTACK] step=313 | reward=-0.0750 | mean(last10): -0.2425
  [ATTACK] step=314 | reward=-0.1000 | mean(last10): -0.2875
  [ATTACK] step=315 | reward=-0.0500 | mean(last10): -0.2425
  [ATTACK] step=316 | reward=-0.4625 | mean(last10): -0.2538
  [ATTACK] step=317 | reward=-0.1500 | mean(last10): -0.2388
  [ATTACK] step=318 | reward=-0.1000 | mean(last10): -0.2338
  [ATTACK] step=319 | reward=-0.4625 | mean(last10): -0.2325
  [ATTACK] step=320 | reward=-0.0625 | mean(last10): -0.2138
  [ATTACK] step=321 | reward=-0.5750 | mean(last10): -0.2512
  [ATTACK] step=322 | reward=+0.2500 | mean(last10): -0.1788
  [ATTACK] step=323 | reward=+0.3000 | mean(last10): -0.1412
  [ATTACK] step=324 | reward=-0.4000 | mean(last10): -0.1712
  [ATTACK] step=325 | reward=-0.1500 | mean(last10): -0.1812
  [ATTACK] step=326 | reward=+0.2000 | mean(last10): -0.1150
  [ATTACK] step=327 | reward=+0.3000 | mean(last10): -0.0700
  [ATTACK] step=328 | reward=+0.0875 | mean(last10): -0.0512
  [ATTACK] step=329 | reward=+0.4375 | mean(last10): +0.0388
  [ATTACK] step=330 | reward=+0.0000 | mean(last10): +0.0450
  [ATTACK] step=331 | reward=+0.4000 | mean(last10): +0.1425
  [ATTACK] step=332 | reward=+0.0125 | mean(last10): +0.1188
  [ATTACK] step=333 | reward=-0.1500 | mean(last10): +0.0737
  [ATTACK] step=334 | reward=-0.7500 | mean(last10): +0.0387
  [ATTACK] step=335 | reward=-0.6000 | mean(last10): -0.0063
  [ATTACK] step=336 | reward=-0.3500 | mean(last10): -0.0613
  [ATTACK] step=337 | reward=-0.0500 | mean(last10): -0.0963
  [ATTACK] step=338 | reward=+0.2500 | mean(last10): -0.0800
  [ATTACK] step=339 | reward=-0.3500 | mean(last10): -0.1588
  [ATTACK] step=340 | reward=+0.2500 | mean(last10): -0.1338
  [ATTACK] step=341 | reward=-0.2500 | mean(last10): -0.1988
  [ATTACK] step=342 | reward=-0.1000 | mean(last10): -0.2100
  [ATTACK] step=343 | reward=-0.4500 | mean(last10): -0.2400
  [ATTACK] step=344 | reward=+0.2000 | mean(last10): -0.1450
  [ATTACK] step=345 | reward=+0.5500 | mean(last10): -0.0300
  [ATTACK] step=346 | reward=-0.2500 | mean(last10): -0.0200
  [ATTACK] step=347 | reward=-0.2125 | mean(last10): -0.0362
  [ATTACK] step=348 | reward=-0.4625 | mean(last10): -0.1075
  [ATTACK] step=349 | reward=-0.2125 | mean(last10): -0.0937
  [ATTACK] step=350 | reward=-0.5250 | mean(last10): -0.1712
  [ATTACK] step=351 | reward=+0.0375 | mean(last10): -0.1425
  [ATTACK] step=352 | reward=+0.2000 | mean(last10): -0.1125
  [ATTACK] step=353 | reward=+0.2000 | mean(last10): -0.0475
  [ATTACK] step=354 | reward=+0.1500 | mean(last10): -0.0525
  [ATTACK] step=355 | reward=+0.3500 | mean(last10): -0.0725
  [ATTACK] step=356 | reward=-0.1000 | mean(last10): -0.0575
  [ATTACK] step=357 | reward=-0.3625 | mean(last10): -0.0725
  [ATTACK] step=358 | reward=+0.0500 | mean(last10): -0.0212
  [ATTACK] step=359 | reward=-0.6000 | mean(last10): -0.0600
  [ATTACK] step=360 | reward=+0.3000 | mean(last10): +0.0225
  [ATTACK] step=361 | reward=-0.2500 | mean(last10): -0.0063
  [ATTACK] step=362 | reward=+0.0500 | mean(last10): -0.0213
  [ATTACK] step=363 | reward=+0.1000 | mean(last10): -0.0313
  [ATTACK] step=364 | reward=+0.0000 | mean(last10): -0.0463
  [ATTACK] step=365 | reward=+0.3000 | mean(last10): -0.0513
  [ATTACK] step=366 | reward=-0.1000 | mean(last10): -0.0513
  [ATTACK] step=367 | reward=+0.3500 | mean(last10): +0.0200
  [ATTACK] step=368 | reward=-0.1500 | mean(last10): -0.0000
  [ATTACK] step=369 | reward=+0.0500 | mean(last10): +0.0650
  [ATTACK] step=370 | reward=+0.1500 | mean(last10): +0.0500
  [ATTACK] step=371 | reward=-0.0625 | mean(last10): +0.0687
  [ATTACK] step=372 | reward=-0.4625 | mean(last10): +0.0175
  [ATTACK] step=373 | reward=-0.2500 | mean(last10): -0.0175
  [ATTACK] step=374 | reward=-0.3000 | mean(last10): -0.0475
  [ATTACK] step=375 | reward=+0.0500 | mean(last10): -0.0725
  [ATTACK] step=376 | reward=-0.2500 | mean(last10): -0.0875
  [ATTACK] step=377 | reward=-0.3625 | mean(last10): -0.1588
  [ATTACK] step=378 | reward=-0.1500 | mean(last10): -0.1588
  [ATTACK] step=379 | reward=+0.1000 | mean(last10): -0.1538
  [ATTACK] step=380 | reward=-0.5000 | mean(last10): -0.2188
  [ATTACK] step=381 | reward=-0.1500 | mean(last10): -0.2275
  [ATTACK] step=382 | reward=+0.1000 | mean(last10): -0.1713
  [ATTACK] step=383 | reward=-0.1500 | mean(last10): -0.1613
  [ATTACK] step=384 | reward=-0.6500 | mean(last10): -0.1963
  [ATTACK] step=385 | reward=-0.2500 | mean(last10): -0.2263
  [ATTACK] step=386 | reward=+0.3000 | mean(last10): -0.1712
  [ATTACK] step=387 | reward=-0.2000 | mean(last10): -0.1550
  [ATTACK] step=388 | reward=+0.3000 | mean(last10): -0.1100
  [ATTACK] step=389 | reward=-0.8000 | mean(last10): -0.2000
  [ATTACK] step=390 | reward=+0.0000 | mean(last10): -0.1500
  [ATTACK] step=391 | reward=+0.2500 | mean(last10): -0.1100
  [ATTACK] step=392 | reward=+0.4500 | mean(last10): -0.0750
  [ATTACK] step=393 | reward=+0.5000 | mean(last10): -0.0100
  [ATTACK] step=394 | reward=-0.3125 | mean(last10): +0.0237
  [ATTACK] step=395 | reward=-0.2000 | mean(last10): +0.0287
  [ATTACK] step=396 | reward=-0.0500 | mean(last10): -0.0063
  [ATTACK] step=397 | reward=-0.1000 | mean(last10): +0.0037
  [ATTACK] step=398 | reward=-0.5000 | mean(last10): -0.0763
  [ATTACK] step=399 | reward=-0.0000 | mean(last10): +0.0037
  [ATTACK] step=400 | reward=-0.1625 | mean(last10): -0.0125
Refreshing datasets (256 episodes x 3 steps)...
==((====))==  Unsloth 2026.4.8: Fast Qwen2 patching. Transformers: 4.56.2. vLLM: 0.9.2.
   \\   /|    Tesla T4. Num GPUs = 2. Max memory: 14.563 GB. Platform: Linux.
O^O/ \_/ \    Torch: 2.7.0+cu126. CUDA: 7.5. CUDA Toolkit: 12.6. Triton: 3.2.0
\        /    Bfloat16 = FALSE. FA [Xformers = 0.0.30. FA2 = False]
 "-____-"     Free license: http://github.com/unslothai/unsloth
Unsloth: Fast downloading is enabled - ignore downloading bars which are red colored!
unsloth/qwen2.5-0.5b-instruct-unsloth-bnb-4bit does not have a padding token! Will use pad_token = <|PAD_TOKEN|>.
   ↳ Refreshed 768 attacker + 768 defender prompts
   ↳ Memory state: attacks=10 defenses=10 breaches=10
Datasets: attacker=256 defender=256
Training Defender (400 steps)...
==((====))==  Unsloth 2026.4.8: Fast Qwen2 patching. Transformers: 4.56.2. vLLM: 0.9.2.
   \\   /|    Tesla T4. Num GPUs = 2. Max memory: 14.563 GB. Platform: Linux.
O^O/ \_/ \    Torch: 2.7.0+cu126. CUDA: 7.5. CUDA Toolkit: 12.6. Triton: 3.2.0
\        /    Bfloat16 = FALSE. FA [Xformers = 0.0.30. FA2 = False]
 "-____-"     Free license: http://github.com/unslothai/unsloth
Unsloth: Fast downloading is enabled - ignore downloading bars which are red colored!
unsloth/qwen2.5-0.5b-instruct-unsloth-bnb-4bit does not have a padding token! Will use pad_token = <|PAD_TOKEN|>.
Unsloth: We now expect `per_device_train_batch_size` * `gradient_accumulation_steps` * `world_size` to be a multiple of `num_generations`.
We will change the batch size of 1 to the `num_generations` of 4

==((====))==  Unsloth - 2x faster free finetuning | Num GPUs used = 1
   \\   /|    Num examples = 256 | Num Epochs = 4 | Total steps = 400
O^O/ \_/ \    Batch size per device = 4 | Gradient accumulation steps = 2
\        /    Data Parallel GPUs = 1 | Total batch size (4 x 2 x 1) = 8
 "-____-"     Trainable parameters = 8,798,208 of 502,830,976 (1.75% trained)

Unsloth: Will smartly offload gradients to save VRAM!

  [DEFEND] step=  1 | reward=-1.0000 | mean(last1): -1.0000
  [DEFEND] step=  2 | reward=-1.0000 | mean(last2): -1.0000
  [DEFEND] step=  3 | reward=-1.0000 | mean(last3): -1.0000
  [DEFEND] step=  4 | reward=-1.0000 | mean(last4): -1.0000
  [DEFEND] step=  5 | reward=-1.0000 | mean(last5): -1.0000
  [DEFEND] step=  6 | reward=-0.6900 | mean(last6): -0.9483
  [DEFEND] step=  7 | reward=-1.0000 | mean(last7): -0.9557
  [DEFEND] step=  8 | reward=-0.6500 | mean(last8): -0.9175
  [DEFEND] step=  9 | reward=-0.8925 | mean(last9): -0.9147
  [DEFEND] step= 10 | reward=-0.9375 | mean(last10): -0.9170
  [DEFEND] step= 11 | reward=-0.9375 | mean(last10): -0.9107
  [DEFEND] step= 12 | reward=-0.8250 | mean(last10): -0.8932
  [DEFEND] step= 13 | reward=-1.0000 | mean(last10): -0.8932
  [DEFEND] step= 14 | reward=-0.7125 | mean(last10): -0.8645
  [DEFEND] step= 15 | reward=-0.8000 | mean(last10): -0.8445
  [DEFEND] step= 16 | reward=-1.0000 | mean(last10): -0.8755
  [DEFEND] step= 17 | reward=-1.0000 | mean(last10): -0.8755
  [DEFEND] step= 18 | reward=-0.5250 | mean(last10): -0.8630
  [DEFEND] step= 19 | reward=-0.8625 | mean(last10): -0.8600
  [DEFEND] step= 20 | reward=-0.7125 | mean(last10): -0.8375
  [DEFEND] step= 21 | reward=-0.7125 | mean(last10): -0.8150
  [DEFEND] step= 22 | reward=-1.0000 | mean(last10): -0.8325
  [DEFEND] step= 23 | reward=-0.9431 | mean(last10): -0.8268
  [DEFEND] step= 24 | reward=-0.8844 | mean(last10): -0.8440
  [DEFEND] step= 25 | reward=-0.9375 | mean(last10): -0.8577
  [DEFEND] step= 26 | reward=-0.9375 | mean(last10): -0.8515
  [DEFEND] step= 27 | reward=-1.0000 | mean(last10): -0.8515
  [DEFEND] step= 28 | reward=-0.9375 | mean(last10): -0.8927
  [DEFEND] step= 29 | reward=-0.7781 | mean(last10): -0.8843
  [DEFEND] step= 30 | reward=-0.6375 | mean(last10): -0.8768
  [DEFEND] step= 31 | reward=-0.8900 | mean(last10): -0.8946
  [DEFEND] step= 32 | reward=-0.9375 | mean(last10): -0.8883
  [DEFEND] step= 33 | reward=-1.0000 | mean(last10): -0.8940
  [DEFEND] step= 34 | reward=-0.6500 | mean(last10): -0.8706
  [DEFEND] step= 35 | reward=-1.0000 | mean(last10): -0.8768
  [DEFEND] step= 36 | reward=-0.9250 | mean(last10): -0.8756
  [DEFEND] step= 37 | reward=-1.0000 | mean(last10): -0.8756
  [DEFEND] step= 38 | reward=-0.8625 | mean(last10): -0.8681
  [DEFEND] step= 39 | reward=-0.9375 | mean(last10): -0.8840
  [DEFEND] step= 40 | reward=-0.9525 | mean(last10): -0.9155
  [DEFEND] step= 41 | reward=-1.0000 | mean(last10): -0.9265
  [DEFEND] step= 42 | reward=-0.4250 | mean(last10): -0.8752
  [DEFEND] step= 43 | reward=-0.9375 | mean(last10): -0.8690
  [DEFEND] step= 44 | reward=-0.9525 | mean(last10): -0.8993
  [DEFEND] step= 45 | reward=-1.0000 | mean(last10): -0.8993
  [DEFEND] step= 46 | reward=-0.6375 | mean(last10): -0.8705
  [DEFEND] step= 47 | reward=-1.0000 | mean(last10): -0.8705
  [DEFEND] step= 48 | reward=-0.7125 | mean(last10): -0.8555
  [DEFEND] step= 49 | reward=-1.0000 | mean(last10): -0.8618
  [DEFEND] step= 50 | reward=-0.5600 | mean(last10): -0.8225
  [DEFEND] step= 51 | reward=-1.0000 | mean(last10): -0.8225
  [DEFEND] step= 52 | reward=-0.9375 | mean(last10): -0.8738
  [DEFEND] step= 53 | reward=-1.0000 | mean(last10): -0.8800
  [DEFEND] step= 54 | reward=-0.9375 | mean(last10): -0.8785
  [DEFEND] step= 55 | reward=-0.8125 | mean(last10): -0.8598
  [DEFEND] step= 56 | reward=-0.5987 | mean(last10): -0.8559
  [DEFEND] step= 57 | reward=-1.0000 | mean(last10): -0.8559
  [DEFEND] step= 58 | reward=-0.9375 | mean(last10): -0.8784
  [DEFEND] step= 59 | reward=-0.9431 | mean(last10): -0.8727
  [DEFEND] step= 60 | reward=-0.8750 | mean(last10): -0.9042
  [DEFEND] step= 61 | reward=-0.9375 | mean(last10): -0.8979
  [DEFEND] step= 62 | reward=-1.0000 | mean(last10): -0.9042
  [DEFEND] step= 63 | reward=-0.8750 | mean(last10): -0.8917
  [DEFEND] step= 64 | reward=-0.9375 | mean(last10): -0.8917
  [DEFEND] step= 65 | reward=-0.6500 | mean(last10): -0.8754
  [DEFEND] step= 66 | reward=-0.9375 | mean(last10): -0.9093
  [DEFEND] step= 67 | reward=-0.6500 | mean(last10): -0.8743
  [DEFEND] step= 68 | reward=-0.9375 | mean(last10): -0.8743
  [DEFEND] step= 69 | reward=-0.5875 | mean(last10): -0.8387
  [DEFEND] step= 70 | reward=-1.0000 | mean(last10): -0.8512
  [DEFEND] step= 71 | reward=-1.0000 | mean(last10): -0.8575
  [DEFEND] step= 72 | reward=-0.8000 | mean(last10): -0.8375
  [DEFEND] step= 73 | reward=-1.0000 | mean(last10): -0.8500
  [DEFEND] step= 74 | reward=-0.6500 | mean(last10): -0.8212
  [DEFEND] step= 75 | reward=-0.9050 | mean(last10): -0.8467
  [DEFEND] step= 76 | reward=-1.0000 | mean(last10): -0.8530
  [DEFEND] step= 77 | reward=-0.8625 | mean(last10): -0.8742
  [DEFEND] step= 78 | reward=-0.9469 | mean(last10): -0.8752
  [DEFEND] step= 79 | reward=-0.8112 | mean(last10): -0.8976
  [DEFEND] step= 80 | reward=-0.6500 | mean(last10): -0.8626
  [DEFEND] step= 81 | reward=-0.5875 | mean(last10): -0.8213
  [DEFEND] step= 82 | reward=-0.4250 | mean(last10): -0.7838
  [DEFEND] step= 83 | reward=-1.0000 | mean(last10): -0.7838
  [DEFEND] step= 84 | reward=-1.0000 | mean(last10): -0.8188
  [DEFEND] step= 85 | reward=-0.8750 | mean(last10): -0.8158
  [DEFEND] step= 86 | reward=-1.0000 | mean(last10): -0.8158
  [DEFEND] step= 87 | reward=-0.5250 | mean(last10): -0.7821
  [DEFEND] step= 88 | reward=-0.7125 | mean(last10): -0.7586
  [DEFEND] step= 89 | reward=-0.5250 | mean(last10): -0.7300
  [DEFEND] step= 90 | reward=-0.4363 | mean(last10): -0.7086
  [DEFEND] step= 91 | reward=-0.8219 | mean(last10): -0.7321
  [DEFEND] step= 92 | reward=-1.0000 | mean(last10): -0.7896
  [DEFEND] step= 93 | reward=-1.0000 | mean(last10): -0.7896
  [DEFEND] step= 94 | reward=-1.0000 | mean(last10): -0.7896
  [DEFEND] step= 95 | reward=-0.9375 | mean(last10): -0.7958
  [DEFEND] step= 96 | reward=-0.9250 | mean(last10): -0.7883
  [DEFEND] step= 97 | reward=-1.0000 | mean(last10): -0.8358
  [DEFEND] step= 98 | reward=-1.0000 | mean(last10): -0.8646
  [DEFEND] step= 99 | reward=-0.9250 | mean(last10): -0.9046
  [DEFEND] step=100 | reward=-0.6500 | mean(last10): -0.9259
  [DEFEND] step=101 | reward=-1.0000 | mean(last10): -0.9437
  [DEFEND] step=102 | reward=-1.0000 | mean(last10): -0.9437
  [DEFEND] step=103 | reward=-0.8500 | mean(last10): -0.9288
  [DEFEND] step=104 | reward=-0.8750 | mean(last10): -0.9163
  [DEFEND] step=105 | reward=-0.7125 | mean(last10): -0.8938
  [DEFEND] step=106 | reward=-0.8219 | mean(last10): -0.8834
  [DEFEND] step=107 | reward=-0.6500 | mean(last10): -0.8484
  [DEFEND] step=108 | reward=-1.0000 | mean(last10): -0.8484
  [DEFEND] step=109 | reward=-0.9050 | mean(last10): -0.8464
  [DEFEND] step=110 | reward=-0.5875 | mean(last10): -0.8402
  [DEFEND] step=111 | reward=-0.8625 | mean(last10): -0.8264
  [DEFEND] step=112 | reward=-0.8775 | mean(last10): -0.8142
  [DEFEND] step=113 | reward=-0.3500 | mean(last10): -0.7642
  [DEFEND] step=114 | reward=-0.8125 | mean(last10): -0.7579
  [DEFEND] step=115 | reward=-0.0750 | mean(last10): -0.6942
  [DEFEND] step=116 | reward=-0.3625 | mean(last10): -0.6482
  [DEFEND] step=117 | reward=-0.9375 | mean(last10): -0.6770
  [DEFEND] step=118 | reward=-1.0000 | mean(last10): -0.6770
  [DEFEND] step=119 | reward=-1.0000 | mean(last10): -0.6865
  [DEFEND] step=120 | reward=-0.9250 | mean(last10): -0.7203
  [DEFEND] step=121 | reward=-0.5987 | mean(last10): -0.6939
  [DEFEND] step=122 | reward=-0.7125 | mean(last10): -0.6774
  [DEFEND] step=123 | reward=-0.6500 | mean(last10): -0.7074
  [DEFEND] step=124 | reward=-0.9525 | mean(last10): -0.7214
  [DEFEND] step=125 | reward=-0.7500 | mean(last10): -0.7889
  [DEFEND] step=126 | reward=-0.6425 | mean(last10): -0.8169
  [DEFEND] step=127 | reward=-0.8406 | mean(last10): -0.8072
  [DEFEND] step=128 | reward=-0.8500 | mean(last10): -0.7922
  [DEFEND] step=129 | reward=-0.6500 | mean(last10): -0.7572
  [DEFEND] step=130 | reward=-0.6875 | mean(last10): -0.7334
  [DEFEND] step=131 | reward=-0.7125 | mean(last10): -0.7448
  [DEFEND] step=132 | reward=-0.8938 | mean(last10): -0.7629
  [DEFEND] step=133 | reward=-0.6500 | mean(last10): -0.7629
  [DEFEND] step=134 | reward=-0.5875 | mean(last10): -0.7264
  [DEFEND] step=135 | reward=-0.8750 | mean(last10): -0.7389
  [DEFEND] step=136 | reward=-0.8000 | mean(last10): -0.7547
  [DEFEND] step=137 | reward=-0.9250 | mean(last10): -0.7631
  [DEFEND] step=138 | reward=-1.0000 | mean(last10): -0.7781
  [DEFEND] step=139 | reward=-0.9375 | mean(last10): -0.8069
  [DEFEND] step=140 | reward=-0.9375 | mean(last10): -0.8319
  [DEFEND] step=141 | reward=-0.8750 | mean(last10): -0.8481
  [DEFEND] step=142 | reward=-0.8625 | mean(last10): -0.8450
  [DEFEND] step=143 | reward=-0.8750 | mean(last10): -0.8675
  [DEFEND] step=144 | reward=-0.7375 | mean(last10): -0.8825
  [DEFEND] step=145 | reward=-0.8813 | mean(last10): -0.8831
  [DEFEND] step=146 | reward=-0.2875 | mean(last10): -0.8319
  [DEFEND] step=147 | reward=-0.8125 | mean(last10): -0.8206
  [DEFEND] step=148 | reward=-0.8450 | mean(last10): -0.8051
  [DEFEND] step=149 | reward=-0.3000 | mean(last10): -0.7414
  [DEFEND] step=150 | reward=-0.6500 | mean(last10): -0.7126
  [DEFEND] step=151 | reward=-0.8313 | mean(last10): -0.7083
  [DEFEND] step=152 | reward=-0.6500 | mean(last10): -0.6870
  [DEFEND] step=153 | reward=-0.0525 | mean(last10): -0.6048
  [DEFEND] step=154 | reward=-0.8750 | mean(last10): -0.6185
  [DEFEND] step=155 | reward=-0.6612 | mean(last10): -0.5965
  [DEFEND] step=156 | reward=-1.0000 | mean(last10): -0.6678
  [DEFEND] step=157 | reward=-1.0000 | mean(last10): -0.6865
  [DEFEND] step=158 | reward=-0.7788 | mean(last10): -0.6799
  [DEFEND] step=159 | reward=-0.6500 | mean(last10): -0.7149
  [DEFEND] step=160 | reward=-0.8625 | mean(last10): -0.7361
  [DEFEND] step=161 | reward=-0.3625 | mean(last10): -0.6892
  [DEFEND] step=162 | reward=-0.0750 | mean(last10): -0.6318
  [DEFEND] step=163 | reward=-0.7125 | mean(last10): -0.6977
  [DEFEND] step=164 | reward=-0.9375 | mean(last10): -0.7040
  [DEFEND] step=165 | reward=-0.6150 | mean(last10): -0.6994
  [DEFEND] step=166 | reward=-0.8750 | mean(last10): -0.6869
  [DEFEND] step=167 | reward=-0.2987 | mean(last10): -0.6167
  [DEFEND] step=168 | reward=-0.6525 | mean(last10): -0.6041
  [DEFEND] step=169 | reward=-0.8750 | mean(last10): -0.6266
  [DEFEND] step=170 | reward=-0.3000 | mean(last10): -0.5704
  [DEFEND] step=171 | reward=-0.6500 | mean(last10): -0.5991
  [DEFEND] step=172 | reward=-0.9375 | mean(last10): -0.6854
  [DEFEND] step=173 | reward=-0.9375 | mean(last10): -0.7079
  [DEFEND] step=174 | reward=-0.1375 | mean(last10): -0.6279
  [DEFEND] step=175 | reward=-0.8125 | mean(last10): -0.6476
  [DEFEND] step=176 | reward=-0.7125 | mean(last10): -0.6314
  [DEFEND] step=177 | reward=-0.8750 | mean(last10): -0.6890
  [DEFEND] step=178 | reward=-0.5125 | mean(last10): -0.6750
  [DEFEND] step=179 | reward=-0.5875 | mean(last10): -0.6463
  [DEFEND] step=180 | reward=-0.8625 | mean(last10): -0.7025
  [DEFEND] step=181 | reward=-0.5250 | mean(last10): -0.6900
  [DEFEND] step=182 | reward=-0.6500 | mean(last10): -0.6612
  [DEFEND] step=183 | reward=-0.8625 | mean(last10): -0.6537
  [DEFEND] step=184 | reward=-0.8181 | mean(last10): -0.7218
  [DEFEND] step=185 | reward=-0.6500 | mean(last10): -0.7056
  [DEFEND] step=186 | reward=-0.8625 | mean(last10): -0.7206
  [DEFEND] step=187 | reward=-0.4625 | mean(last10): -0.6793
  [DEFEND] step=188 | reward=-0.8125 | mean(last10): -0.7093
  [DEFEND] step=189 | reward=-0.3625 | mean(last10): -0.6868
  [DEFEND] step=190 | reward=-0.8750 | mean(last10): -0.6881
  [DEFEND] step=191 | reward=-0.3000 | mean(last10): -0.6656
  [DEFEND] step=192 | reward=-0.6025 | mean(last10): -0.6608
  [DEFEND] step=193 | reward=-0.8625 | mean(last10): -0.6608
  [DEFEND] step=194 | reward=-0.4375 | mean(last10): -0.6228
  [DEFEND] step=195 | reward=-0.8450 | mean(last10): -0.6423
  [DEFEND] step=196 | reward=-0.8125 | mean(last10): -0.6373
  [DEFEND] step=197 | reward=-0.8863 | mean(last10): -0.6796
  [DEFEND] step=198 | reward=-0.9375 | mean(last10): -0.6921
  [DEFEND] step=199 | reward=-0.8000 | mean(last10): -0.7359
  [DEFEND] step=200 | reward=-0.9375 | mean(last10): -0.7421
  [DEFEND] step=201 | reward=-0.7550 | mean(last10): -0.7876
  [DEFEND] step=202 | reward=-0.8500 | mean(last10): -0.8124
  [DEFEND] step=203 | reward=-0.7500 | mean(last10): -0.8011
  [DEFEND] step=204 | reward=-0.7550 | mean(last10): -0.8329
  [DEFEND] step=205 | reward=-0.9375 | mean(last10): -0.8421
  [DEFEND] step=206 | reward=-0.4250 | mean(last10): -0.8034
  [DEFEND] step=207 | reward=-0.7031 | mean(last10): -0.7851
  [DEFEND] step=208 | reward=-0.9525 | mean(last10): -0.7866
  [DEFEND] step=209 | reward=-0.3000 | mean(last10): -0.7366
  [DEFEND] step=210 | reward=-0.8125 | mean(last10): -0.7241
  [DEFEND] step=211 | reward=-0.6556 | mean(last10): -0.7141
  [DEFEND] step=212 | reward=-0.5875 | mean(last10): -0.6879
  [DEFEND] step=213 | reward=-0.8625 | mean(last10): -0.6991
  [DEFEND] step=214 | reward=-0.8000 | mean(last10): -0.7036
  [DEFEND] step=215 | reward=-0.8056 | mean(last10): -0.6904
  [DEFEND] step=216 | reward=-0.2375 | mean(last10): -0.6717
  [DEFEND] step=217 | reward=-0.5800 | mean(last10): -0.6594
  [DEFEND] step=218 | reward=-0.8900 | mean(last10): -0.6531
  [DEFEND] step=219 | reward=-0.9375 | mean(last10): -0.7169
  [DEFEND] step=220 | reward=-0.6556 | mean(last10): -0.7012
  [DEFEND] step=221 | reward=-0.6500 | mean(last10): -0.7006
  [DEFEND] step=222 | reward=-0.2125 | mean(last10): -0.6631
  [DEFEND] step=223 | reward=-0.5250 | mean(last10): -0.6294
  [DEFEND] step=224 | reward=-0.8175 | mean(last10): -0.6311
  [DEFEND] step=225 | reward=-0.5225 | mean(last10): -0.6028
  [DEFEND] step=226 | reward=-0.2250 | mean(last10): -0.6016
  [DEFEND] step=227 | reward=-0.8350 | mean(last10): -0.6271
  [DEFEND] step=228 | reward=-0.4500 | mean(last10): -0.5831
  [DEFEND] step=229 | reward=-0.3175 | mean(last10): -0.5211
  [DEFEND] step=230 | reward=-0.8800 | mean(last10): -0.5435
  [DEFEND] step=231 | reward=-0.4875 | mean(last10): -0.5273
  [DEFEND] step=232 | reward=-0.7306 | mean(last10): -0.5791
  [DEFEND] step=233 | reward=-0.6500 | mean(last10): -0.5916
  [DEFEND] step=234 | reward=-0.8750 | mean(last10): -0.5973
  [DEFEND] step=235 | reward=-0.7375 | mean(last10): -0.6188
  [DEFEND] step=236 | reward=-0.5350 | mean(last10): -0.6498
  [DEFEND] step=237 | reward=-0.3625 | mean(last10): -0.6026
  [DEFEND] step=238 | reward=-0.4675 | mean(last10): -0.6043
  [DEFEND] step=239 | reward=-0.2125 | mean(last10): -0.5938
  [DEFEND] step=240 | reward=-0.6675 | mean(last10): -0.5726
  [DEFEND] step=241 | reward=-0.8500 | mean(last10): -0.6088
  [DEFEND] step=242 | reward=-0.9250 | mean(last10): -0.6283
  [DEFEND] step=243 | reward=-0.6938 | mean(last10): -0.6326
  [DEFEND] step=244 | reward=-0.6550 | mean(last10): -0.6106
  [DEFEND] step=245 | reward=-0.4625 | mean(last10): -0.5831
  [DEFEND] step=246 | reward=-0.6219 | mean(last10): -0.5918
  [DEFEND] step=247 | reward=-0.4625 | mean(last10): -0.6018
  [DEFEND] step=248 | reward=-0.8331 | mean(last10): -0.6384
  [DEFEND] step=249 | reward=-0.9050 | mean(last10): -0.7076
  [DEFEND] step=250 | reward=-0.8500 | mean(last10): -0.7259
  [DEFEND] step=251 | reward=-0.0406 | mean(last10): -0.6449
  [DEFEND] step=252 | reward=-0.5875 | mean(last10): -0.6112
  [DEFEND] step=253 | reward=-0.4375 | mean(last10): -0.5856
  [DEFEND] step=254 | reward=-0.7975 | mean(last10): -0.5998
  [DEFEND] step=255 | reward=-0.6500 | mean(last10): -0.6186
  [DEFEND] step=256 | reward=-0.6975 | mean(last10): -0.6261
  [DEFEND] step=257 | reward=+0.0500 | mean(last10): -0.5749
  [DEFEND] step=258 | reward=-0.7500 | mean(last10): -0.5666
  [DEFEND] step=259 | reward=-0.6750 | mean(last10): -0.5436
  [DEFEND] step=260 | reward=-0.5987 | mean(last10): -0.5184
  [DEFEND] step=261 | reward=-0.8181 | mean(last10): -0.5962
  [DEFEND] step=262 | reward=-0.6350 | mean(last10): -0.6009
  [DEFEND] step=263 | reward=-0.5425 | mean(last10): -0.6114
  [DEFEND] step=264 | reward=-0.7125 | mean(last10): -0.6029
  [DEFEND] step=265 | reward=-0.5250 | mean(last10): -0.5904
  [DEFEND] step=266 | reward=-0.2344 | mean(last10): -0.5441
  [DEFEND] step=267 | reward=-0.5094 | mean(last10): -0.6001
  [DEFEND] step=268 | reward=-0.8406 | mean(last10): -0.6091
  [DEFEND] step=269 | reward=-0.6750 | mean(last10): -0.6091
  [DEFEND] step=270 | reward=-0.0844 | mean(last10): -0.5577
  [DEFEND] step=271 | reward=-0.0875 | mean(last10): -0.4846
  [DEFEND] step=272 | reward=-0.7825 | mean(last10): -0.4994
  [DEFEND] step=273 | reward=-0.6625 | mean(last10): -0.5114
  [DEFEND] step=274 | reward=-0.2250 | mean(last10): -0.4626
  [DEFEND] step=275 | reward=-0.1250 | mean(last10): -0.4226
  [DEFEND] step=276 | reward=-0.4450 | mean(last10): -0.4437
  [DEFEND] step=277 | reward=-0.4500 | mean(last10): -0.4378
  [DEFEND] step=278 | reward=-0.6438 | mean(last10): -0.4181
  [DEFEND] step=279 | reward=-0.5000 | mean(last10): -0.4006
  [DEFEND] step=280 | reward=-0.6125 | mean(last10): -0.4534
  [DEFEND] step=281 | reward=-0.7500 | mean(last10): -0.5196
  [DEFEND] step=282 | reward=-0.3750 | mean(last10): -0.4789
  [DEFEND] step=283 | reward=-0.2325 | mean(last10): -0.4359
  [DEFEND] step=284 | reward=-0.7125 | mean(last10): -0.4846
  [DEFEND] step=285 | reward=-1.0000 | mean(last10): -0.5721
  [DEFEND] step=286 | reward=-0.6925 | mean(last10): -0.5969
  [DEFEND] step=287 | reward=-0.8125 | mean(last10): -0.6331
  [DEFEND] step=288 | reward=-0.6875 | mean(last10): -0.6375
  [DEFEND] step=289 | reward=-0.4737 | mean(last10): -0.6349
  [DEFEND] step=290 | reward=-0.2875 | mean(last10): -0.6024
  [DEFEND] step=291 | reward=-0.6625 | mean(last10): -0.5936
  [DEFEND] step=292 | reward=+0.1625 | mean(last10): -0.5399
  [DEFEND] step=293 | reward=-0.8775 | mean(last10): -0.6044
  [DEFEND] step=294 | reward=-0.6919 | mean(last10): -0.6023
  [DEFEND] step=295 | reward=-0.2656 | mean(last10): -0.5289
  [DEFEND] step=296 | reward=-0.0600 | mean(last10): -0.4656
  [DEFEND] step=297 | reward=-0.3300 | mean(last10): -0.4174
  [DEFEND] step=298 | reward=-0.7375 | mean(last10): -0.4224
  [DEFEND] step=299 | reward=-0.2250 | mean(last10): -0.3975
  [DEFEND] step=300 | reward=+0.1125 | mean(last10): -0.3575
  [DEFEND] step=301 | reward=-0.8806 | mean(last10): -0.3793
  [DEFEND] step=302 | reward=+0.0625 | mean(last10): -0.3893
  [DEFEND] step=303 | reward=-0.4625 | mean(last10): -0.3478
  [DEFEND] step=304 | reward=-0.8125 | mean(last10): -0.3599
  [DEFEND] step=305 | reward=+0.3375 | mean(last10): -0.2996
  [DEFEND] step=306 | reward=-0.4375 | mean(last10): -0.3373
  [DEFEND] step=307 | reward=-0.4950 | mean(last10): -0.3538
  [DEFEND] step=308 | reward=-0.5125 | mean(last10): -0.3313
  [DEFEND] step=309 | reward=-0.3500 | mean(last10): -0.3438
  [DEFEND] step=310 | reward=-0.8650 | mean(last10): -0.4416
  [DEFEND] step=311 | reward=-0.7100 | mean(last10): -0.4245
  [DEFEND] step=312 | reward=+0.2875 | mean(last10): -0.4020
  [DEFEND] step=313 | reward=-0.6300 | mean(last10): -0.4188
  [DEFEND] step=314 | reward=-0.0000 | mean(last10): -0.3375
  [DEFEND] step=315 | reward=-0.6125 | mean(last10): -0.4325
  [DEFEND] step=316 | reward=-0.4625 | mean(last10): -0.4350
  [DEFEND] step=317 | reward=-0.6750 | mean(last10): -0.4530
  [DEFEND] step=318 | reward=+0.2250 | mean(last10): -0.3793
  [DEFEND] step=319 | reward=+0.0625 | mean(last10): -0.3380
  [DEFEND] step=320 | reward=-0.7875 | mean(last10): -0.3303
  [DEFEND] step=321 | reward=-0.2375 | mean(last10): -0.2830
  [DEFEND] step=322 | reward=-0.5125 | mean(last10): -0.3630
  [DEFEND] step=323 | reward=-0.7325 | mean(last10): -0.3733
  [DEFEND] step=324 | reward=-0.3375 | mean(last10): -0.4070
  [DEFEND] step=325 | reward=+0.3375 | mean(last10): -0.3120
  [DEFEND] step=326 | reward=-0.1625 | mean(last10): -0.2820
  [DEFEND] step=327 | reward=-0.8125 | mean(last10): -0.2958
  [DEFEND] step=328 | reward=-0.2875 | mean(last10): -0.3470
  [DEFEND] step=329 | reward=-0.2250 | mean(last10): -0.3758
  [DEFEND] step=330 | reward=-0.5875 | mean(last10): -0.3558
  [DEFEND] step=331 | reward=-0.6869 | mean(last10): -0.4007
  [DEFEND] step=332 | reward=-0.4625 | mean(last10): -0.3957
  [DEFEND] step=333 | reward=-0.2044 | mean(last10): -0.3429
  [DEFEND] step=334 | reward=-0.6750 | mean(last10): -0.3766
  [DEFEND] step=335 | reward=-0.4500 | mean(last10): -0.4554
  [DEFEND] step=336 | reward=-0.8125 | mean(last10): -0.5204
  [DEFEND] step=337 | reward=-0.0750 | mean(last10): -0.4466
  [DEFEND] step=338 | reward=-0.8150 | mean(last10): -0.4994
  [DEFEND] step=339 | reward=+0.0500 | mean(last10): -0.4719
  [DEFEND] step=340 | reward=-0.1919 | mean(last10): -0.4323
  [DEFEND] step=341 | reward=-0.7375 | mean(last10): -0.4374
  [DEFEND] step=342 | reward=-0.7437 | mean(last10): -0.4655
  [DEFEND] step=343 | reward=-0.2125 | mean(last10): -0.4663
  [DEFEND] step=344 | reward=-0.5375 | mean(last10): -0.4526
  [DEFEND] step=345 | reward=+0.2575 | mean(last10): -0.3818
  [DEFEND] step=346 | reward=-0.4338 | mean(last10): -0.3439
  [DEFEND] step=347 | reward=-0.6350 | mean(last10): -0.3999
  [DEFEND] step=348 | reward=-0.6025 | mean(last10): -0.3787
  [DEFEND] step=349 | reward=-0.1125 | mean(last10): -0.3949
  [DEFEND] step=350 | reward=-0.4250 | mean(last10): -0.4183
  [DEFEND] step=351 | reward=-0.3750 | mean(last10): -0.3820
  [DEFEND] step=352 | reward=-0.8175 | mean(last10): -0.3894
  [DEFEND] step=353 | reward=-0.7250 | mean(last10): -0.4406
  [DEFEND] step=354 | reward=-0.5750 | mean(last10): -0.4444
  [DEFEND] step=355 | reward=-0.3750 | mean(last10): -0.5076
  [DEFEND] step=356 | reward=-0.5125 | mean(last10): -0.5155
  [DEFEND] step=357 | reward=-0.8000 | mean(last10): -0.5320
  [DEFEND] step=358 | reward=-0.3025 | mean(last10): -0.5020
  [DEFEND] step=359 | reward=-0.1000 | mean(last10): -0.5008
  [DEFEND] step=360 | reward=+0.0625 | mean(last10): -0.4520
  [DEFEND] step=361 | reward=-0.7250 | mean(last10): -0.4870
  [DEFEND] step=362 | reward=-0.5362 | mean(last10): -0.4589
  [DEFEND] step=363 | reward=+0.3625 | mean(last10): -0.3501
  [DEFEND] step=364 | reward=-0.7500 | mean(last10): -0.3676
  [DEFEND] step=365 | reward=-0.6100 | mean(last10): -0.3911
  [DEFEND] step=366 | reward=-0.7600 | mean(last10): -0.4159
  [DEFEND] step=367 | reward=-0.3650 | mean(last10): -0.3724
  [DEFEND] step=368 | reward=-0.7788 | mean(last10): -0.4200
  [DEFEND] step=369 | reward=-0.5750 | mean(last10): -0.4675
  [DEFEND] step=370 | reward=-0.7319 | mean(last10): -0.5469
  [DEFEND] step=371 | reward=-0.2250 | mean(last10): -0.4969
  [DEFEND] step=372 | reward=-0.5537 | mean(last10): -0.4987
  [DEFEND] step=373 | reward=-0.3250 | mean(last10): -0.5674
  [DEFEND] step=374 | reward=-0.7931 | mean(last10): -0.5717
  [DEFEND] step=375 | reward=-0.3000 | mean(last10): -0.5407
  [DEFEND] step=376 | reward=+0.1375 | mean(last10): -0.4510
  [DEFEND] step=377 | reward=-0.4375 | mean(last10): -0.4582
  [DEFEND] step=378 | reward=-0.6125 | mean(last10): -0.4416
  [DEFEND] step=379 | reward=+0.0625 | mean(last10): -0.3779
  [DEFEND] step=380 | reward=-0.4044 | mean(last10): -0.3451
  [DEFEND] step=381 | reward=-0.6250 | mean(last10): -0.3851
  [DEFEND] step=382 | reward=-0.7044 | mean(last10): -0.4002
  [DEFEND] step=383 | reward=+0.0250 | mean(last10): -0.3652
  [DEFEND] step=384 | reward=-0.5000 | mean(last10): -0.3359
  [DEFEND] step=385 | reward=-0.3625 | mean(last10): -0.3421
  [DEFEND] step=386 | reward=-0.2625 | mean(last10): -0.3821
  [DEFEND] step=387 | reward=+0.1125 | mean(last10): -0.3271
  [DEFEND] step=388 | reward=+0.1500 | mean(last10): -0.2509
  [DEFEND] step=389 | reward=-0.4500 | mean(last10): -0.3021
  [DEFEND] step=390 | reward=-0.5325 | mean(last10): -0.3149
  [DEFEND] step=391 | reward=-0.7075 | mean(last10): -0.3232
  [DEFEND] step=392 | reward=-0.6125 | mean(last10): -0.3140
  [DEFEND] step=393 | reward=-0.4325 | mean(last10): -0.3598
  [DEFEND] step=394 | reward=-0.3250 | mean(last10): -0.3423
  [DEFEND] step=395 | reward=-0.8000 | mean(last10): -0.3860
  [DEFEND] step=396 | reward=-0.1625 | mean(last10): -0.3760
  [DEFEND] step=397 | reward=-0.8175 | mean(last10): -0.4690
  [DEFEND] step=398 | reward=-0.5250 | mean(last10): -0.5365
  [DEFEND] step=399 | reward=+0.1344 | mean(last10): -0.4781
  [DEFEND] step=400 | reward=-0.5906 | mean(last10): -0.4839

Quick rollout (trained attacker vs trained defender):
==((====))==  Unsloth 2026.4.8: Fast Qwen2 patching. Transformers: 4.56.2. vLLM: 0.9.2.
   \\   /|    Tesla T4. Num GPUs = 2. Max memory: 14.563 GB. Platform: Linux.
O^O/ \_/ \    Torch: 2.7.0+cu126. CUDA: 7.5. CUDA Toolkit: 12.6. Triton: 3.2.0
\        /    Bfloat16 = FALSE. FA [Xformers = 0.0.30. FA2 = False]
 "-____-"     Free license: http://github.com/unslothai/unsloth
Unsloth: Fast downloading is enabled - ignore downloading bars which are red colored!
unsloth/qwen2.5-0.5b-instruct-unsloth-bnb-4bit does not have a padding token! Will use pad_token = <|PAD_TOKEN|>.
==((====))==  Unsloth 2026.4.8: Fast Qwen2 patching. Transformers: 4.56.2. vLLM: 0.9.2.
   \\   /|    Tesla T4. Num GPUs = 2. Max memory: 14.563 GB. Platform: Linux.
O^O/ \_/ \    Torch: 2.7.0+cu126. CUDA: 7.5. CUDA Toolkit: 12.6. Triton: 3.2.0
\        /    Bfloat16 = FALSE. FA [Xformers = 0.0.30. FA2 = False]
 "-____-"     Free license: http://github.com/unslothai/unsloth
Unsloth: Fast downloading is enabled - ignore downloading bars which are red colored!
unsloth/qwen2.5-0.5b-instruct-unsloth-bnb-4bit does not have a padding token! Will use pad_token = <|PAD_TOKEN|>.

--- Episode 1 | scenario=bulk_phish | weakness=PHISH ---
  Step 1: Atk=PHISH        | Def=PATCH        | BREACHED ⚠ | Health=85% | Chains=['LPE', 'RANSOM']
  Step 2: Atk=PHISH        | Def=PATCH        | BREACHED ⚠ | Health=70% | Chains=['LPE', 'RANSOM']
  Step 3: Atk=PHISH        | Def=PATCH        | BREACHED ⚠ | Health=55% | Chains=['LPE', 'RANSOM']
  → Final health: 55% (DEGRADED)

--- Episode 2 | scenario=backup_gap | weakness=RANSOM ---
  Step 1: Atk=RANSOM       | Def=MFA          | BREACHED ⚠ | Health=60% | Chains=[]
  Step 2: Atk=RANSOM       | Def=PATCH        | BREACHED ⚠ | Health=20% | Chains=[]
  Step 3: Atk=RANSOM       | Def=PATCH        | BREACHED ⚠ | Health=0% | Chains=[]
  → Final health: 0% (CRITICAL)

--- Episode 3 | scenario=ssh_brute | weakness=BRUTEFORCE ---
  Step 1: Atk=RANSOM       | Def=MFA          | BLOCKED ✓ | Health=100% | Chains=[]
  Step 2: Atk=RANSOM       | Def=MFA          | BLOCKED ✓ | Health=100% | Chains=[]
  Step 3: Atk=RANSOM       | Def=MFA          | BLOCKED ✓ | Health=100% | Chains=[]
  → Final health: 100% (STABLE)
Breach rate: 44.00% (11/25) | Chain activations: 21
==((====))==  Unsloth 2026.4.8: Fast Qwen2 patching. Transformers: 4.56.2. vLLM: 0.9.2.
   \\   /|    Tesla T4. Num GPUs = 2. Max memory: 14.563 GB. Platform: Linux.
O^O/ \_/ \    Torch: 2.7.0+cu126. CUDA: 7.5. CUDA Toolkit: 12.6. Triton: 3.2.0
\        /    Bfloat16 = FALSE. FA [Xformers = 0.0.30. FA2 = False]
 "-____-"     Free license: http://github.com/unslothai/unsloth
Unsloth: Fast downloading is enabled - ignore downloading bars which are red colored!
Unsloth: QLoRA and full finetuning all not selected. Switching to 16bit LoRA.

unsloth/Qwen2.5-0.5B-Instruct does not have a padding token! Will use pad_token = <|PAD_TOKEN|>.
Tokenizer saved to attacker/defender output dirs
Training complete
Attacker checkpoint: outputs/grpo_training/attacker
Defender checkpoint: outputs/grpo_training/defender

import matplotlib.pyplot as plt
import numpy as np

res = training_results["standard"]

def smooth(values, w=20):
    arr = np.asarray(values, dtype=float)
    if len(arr) < w:
        return arr
    return np.convolve(arr, np.ones(w)/w, mode="valid")

w = 20

# ── Plot 1: individual reward curves ─────────────────────────────────────────
fig, axes = plt.subplots(1, 2, figsize=(16, 5))
fig.patch.set_facecolor("#0d0d1a")
for ax in axes:
    ax.set_facecolor("#0d0d1a")
    ax.tick_params(colors="white")
    for s in ax.spines.values(): s.set_color("#444")
    ax.grid(True, alpha=0.12, color="white")

def plot_curve(ax, steps, values, title, color):
    if not steps or not values:
        return
    steps_arr  = np.array(steps)
    values_arr = np.array(values, dtype=float)
    ax.plot(steps_arr, values_arr, color=color, alpha=0.25, linewidth=1, label="Raw")
    sm = smooth(values_arr, w)
    if len(sm) > 0:
        ax.plot(steps_arr[w-1:], sm, color=color, linewidth=2.5, label=f"Rolling mean (w={w})")
    if len(steps_arr) > 2:
        z = np.polyfit(steps_arr, values_arr, 1)
        ax.plot(steps_arr, np.poly1d(z)(steps_arr), color="white",
                linestyle=":", linewidth=1, alpha=0.7, label=f"Trend ({z[0]:+.4f}/step)")
    ax.axhline(0, color="white", linewidth=0.4, alpha=0.3)
    ax.set_title(title, color="white", fontweight="bold", fontsize=12)
    ax.set_xlabel("Training Step", color="white", fontsize=10)
    ax.set_ylabel("Reward", color="white", fontsize=10)
    ax.legend(facecolor="#0d0d1a", edgecolor="#444", labelcolor="white", fontsize=8)

plot_curve(axes[0], res["atk_steps"], res["atk_rewards"], "Attacker Reward (GRPO)", "#C875FF")
plot_curve(axes[1], res["steps"],     res["rewards"],     "Defender Reward (GRPO)", "#00C896")

plt.suptitle(f"GRPO Training — {TRAIN_MODEL} | {MAX_STEPS} def steps + {ATTACKER_STEPS} atk steps",
             color="white", fontsize=13, fontweight="bold")
plt.tight_layout()
plt.savefig(OUTPUT_DIR / "training_curves.png", dpi=150, facecolor=fig.get_facecolor())
plt.show()
print("Saved training_curves.png")

# ── Plot 2: arms-race — both on same axes ─────────────────────────────────────
fig2, ax2 = plt.subplots(figsize=(13, 5))
fig2.patch.set_facecolor("#0d0d1a")
ax2.set_facecolor("#0d0d1a")
ax2.tick_params(colors="white")
for s in ax2.spines.values(): s.set_color("#444")
ax2.grid(True, alpha=0.12, color="white")

atk_arr = np.array(res["atk_rewards"], dtype=float)
def_arr = np.array(res["rewards"],     dtype=float)
atk_s   = np.array(res["atk_steps"])
def_s   = np.array(res["steps"])

ax2.plot(atk_s, atk_arr, color="#C875FF", alpha=0.15, linewidth=1)
ax2.plot(def_s, def_arr, color="#00C896", alpha=0.15, linewidth=1)

if len(atk_arr) >= w:
    ax2.plot(atk_s[w-1:], smooth(atk_arr, w), color="#C875FF",
             linewidth=2.5, label="Attacker (rolling mean)")
if len(def_arr) >= w:
    ax2.plot(def_s[w-1:], smooth(def_arr, w), color="#00C896",
             linewidth=2.5, label="Defender (rolling mean)")

ax2.axhline(0, color="white", linewidth=0.4, alpha=0.3)
ax2.set_xlabel("Training Step", color="white", fontsize=11)
ax2.set_ylabel("Reward",        color="white", fontsize=11)
ax2.set_title(
    "Arms Race — Attacker vs Defender Reward\n"
    "When Defender improves, Attacker adapts — and vice versa",
    color="white", fontsize=12, fontweight="bold")
ax2.legend(facecolor="#0d0d1a", edgecolor="#444", labelcolor="white", fontsize=10)
plt.tight_layout()
plt.savefig(OUTPUT_DIR / "arms_race.png", dpi=150, facecolor=fig2.get_facecolor())
plt.show()
print("Saved arms_race.png")

Saved training_curves.png

Saved arms_race.png

print("Saved attacker:", training_results["standard"]["atk_dir"])
print("Saved defender:", training_results["standard"]["output_dir"])

Saved attacker: outputs/grpo_training/attacker
Saved defender: outputs/grpo_training/defender

# ============================================================
# Total Notebook Runtime
# ============================================================
import time
elapsed_s = time.time() - NB_START_TIME
print(f"\nTotal notebook runtime: {elapsed_s/60:.2f} minutes ({elapsed_s:.1f} seconds)")

Total notebook runtime: 133.89 minutes (8033.2 seconds)

import numpy as np
import matplotlib.pyplot as plt

def rolling_mean(x, w=20):
    x = np.asarray(x, dtype=float)
    if len(x) < w:
        return x
    kernel = np.ones(w) / w
    return np.convolve(x, kernel, mode="valid")

w = 20

atk_steps = np.array(res["atk_steps"])
atk_rewards = np.array(res["atk_rewards"])
def_steps = np.array(res["steps"])
def_rewards = np.array(res["rewards"])

atk_rm = rolling_mean(atk_rewards, w)
def_rm = rolling_mean(def_rewards, w)

fig, ax = plt.subplots(figsize=(12, 5))

# Raw (faint)
ax.plot(atk_steps, atk_rewards, color="#C875FF", alpha=0.25, label="Attacker raw")
ax.plot(def_steps, def_rewards, color="#00C896", alpha=0.25, label="Defender raw")

# Rolling mean (bold). Note x-axis shifts because rolling_mean shortens length.
ax.plot(atk_steps[w-1:], atk_rm, color="#C875FF", linewidth=2.5, label=f"Attacker rolling mean (w={w})")
ax.plot(def_steps[w-1:], def_rm, color="#00C896", linewidth=2.5, label=f"Defender rolling mean (w={w})")

ax.set_title("Training reward: raw + rolling mean")
ax.set_xlabel("Step")
ax.set_ylabel("Reward")
ax.grid(True, alpha=0.2)
ax.legend()
plt.show()

!pip -q install huggingface_hub

from huggingface_hub import HfApi
import os

HF_TOKEN = "hf_<REDACTED>"
os.environ["HF_TOKEN"] = HF_TOKEN
if HF_TOKEN:
    os.environ["HF_TOKEN"] = HF_TOKEN
else:
    pass

api = HfApi()

attacker_dir = str(atk_out)  
defender_dir = str(def_out)   

attacker_repo = "RapidOrc121/IR_attacker"
defender_repo = "RapidOrc121/IR_defender"

if attacker_repo and HF_TOKEN:
    api.upload_folder(
        folder_path=attacker_dir,
        repo_id=attacker_repo,
        repo_type="model",
        commit_message="Upload attacker GRPO checkpoint",
    )
    print("Uploaded attacker")
else:
    pass

Uploaded attacker

if defender_repo and HF_TOKEN:
    api.upload_folder(
        folder_path=defender_dir,
        repo_id=defender_repo,
        repo_type="model",
        commit_message="Upload defender GRPO checkpoint",
    )
    print("Uploaded defender")
else:
    pass

Uploaded defender

import random
import numpy as np

EVAL_N = int(EVAL_EPISODES) if "EVAL_EPISODES" in globals() else 50
EVAL_SEED = 1234

# For local eval, we use the saved checkpoints from this run.
# Baseline always follows your top config (`TRAIN_MODEL` → `TRAIN_MODEL_ID`).
EVAL_ATTACKER_CKPT = str(atk_out)
EVAL_DEFENDER_CKPT = str(def_out)
EVAL_BASELINE_CKPT = BASELINE_MODEL if "BASELINE_MODEL" in globals() else TRAIN_MODEL_ID

# No prints here; eval summary is printed in the next cells.

from dataclasses import dataclass

@dataclass
class EvalResult:
    mean_reward: float
    std_reward: float
    max_reward: float
    format_valid_rate: float


def _to_eval_result(rewards: list[float], valid_flags: list[int]) -> EvalResult:
    r = np.asarray(rewards, dtype=float)
    v = np.asarray(valid_flags, dtype=float)
    return EvalResult(
        mean_reward=float(r.mean()) if len(r) else 0.0,
        std_reward=float(r.std(ddof=0)) if len(r) else 0.0,
        max_reward=float(r.max()) if len(r) else 0.0,
        format_valid_rate=float(v.mean()) if len(v) else 0.0,
    )


@torch.no_grad()
def eval_attacker_one_step(model_ckpt: str, n=20, seed=1234, temperature=0.4) -> EvalResult:
    random.seed(seed)
    np.random.seed(seed)

    m, tok = FastModel.from_pretrained(
        model_name=model_ckpt, max_seq_length=MAX_SEQ_LENGTH, load_in_4bit=True
    )

    rewards: list[float] = []
    valid: list[int] = []

    for _ in range(n):
        sc = ENV.sample_scenario()
        ep = ENV.new_episode(sc)
        prompt = ENV.attacker_prompt(sc, ep, attacker_memory_line())
        out = sample_actions(m, tok, [prompt], temperature=temperature)[0]
        atk = parse_attack(out)

        is_valid = int(atk in ATTACKS)
        valid.append(is_valid)

        if not is_valid:
            rewards.append(float(FORMAT_PENALTY))
        else:
            rewards.append(1.0 if atk == sc["weakness"] else -0.2)

    m.cpu(); del m
    torch.cuda.empty_cache()

    return _to_eval_result(rewards, valid)


@torch.no_grad()
def eval_defender_one_step(model_ckpt: str, n=20, seed=1234, temperature=0.4) -> EvalResult:
    random.seed(seed)
    np.random.seed(seed)

    m, tok = FastModel.from_pretrained(
        model_name=model_ckpt, max_seq_length=MAX_SEQ_LENGTH, load_in_4bit=True
    )

    rewards: list[float] = []
    valid: list[int] = []

    for _ in range(n):
        sc = ENV.sample_scenario()
        ep = ENV.new_episode(sc)
        atk = sc["weakness"]
        prompt = ENV.defender_prompt(sc, ep, atk, defender_memory_lines())
        out = sample_actions(m, tok, [prompt], temperature=temperature)[0]
        df = parse_defense(out)

        is_valid = int(df in DEFENSES)
        valid.append(is_valid)

        if not is_valid:
            rewards.append(float(FORMAT_PENALTY))
        else:
            corr = get_counter(sc)
            rewards.append(1.0 if df == corr else -1.0)

    m.cpu(); del m
    torch.cuda.empty_cache()

    return _to_eval_result(rewards, valid)


def as_row(r: EvalResult) -> dict:
    return {
        "mean_reward": r.mean_reward,
        "std_reward": r.std_reward,
        "max_reward": r.max_reward,
        "format_valid_rate": r.format_valid_rate,
    }

import pandas as pd

att_base = eval_attacker_one_step(EVAL_BASELINE_CKPT, n=EVAL_N, seed=EVAL_SEED, temperature=0.4)
att_tr   = eval_attacker_one_step(EVAL_ATTACKER_CKPT,  n=EVAL_N, seed=EVAL_SEED, temperature=0.4)

def_base = eval_defender_one_step(EVAL_BASELINE_CKPT, n=EVAL_N, seed=EVAL_SEED, temperature=0.4)
def_tr   = eval_defender_one_step(EVAL_DEFENDER_CKPT,  n=EVAL_N, seed=EVAL_SEED, temperature=0.4)

rows = {
    "attacker_baseline": as_row(att_base),
    "attacker_trained":  as_row(att_tr),
    "defender_baseline": as_row(def_base),
    "defender_trained":  as_row(def_tr),
}

df = pd.DataFrame.from_dict(rows, orient="index")
df

==((====))==  Unsloth 2026.4.8: Fast Qwen2 patching. Transformers: 4.56.2. vLLM: 0.9.2.
   \\   /|    Tesla T4. Num GPUs = 2. Max memory: 14.563 GB. Platform: Linux.
O^O/ \_/ \    Torch: 2.7.0+cu126. CUDA: 7.5. CUDA Toolkit: 12.6. Triton: 3.2.0
\        /    Bfloat16 = FALSE. FA [Xformers = 0.0.30. FA2 = False]
 "-____-"     Free license: http://github.com/unslothai/unsloth
Unsloth: Fast downloading is enabled - ignore downloading bars which are red colored!
unsloth/qwen2.5-0.5b-instruct-unsloth-bnb-4bit does not have a padding token! Will use pad_token = <|PAD_TOKEN|>.
==((====))==  Unsloth 2026.4.8: Fast Qwen2 patching. Transformers: 4.56.2. vLLM: 0.9.2.
   \\   /|    Tesla T4. Num GPUs = 2. Max memory: 14.563 GB. Platform: Linux.
O^O/ \_/ \    Torch: 2.7.0+cu126. CUDA: 7.5. CUDA Toolkit: 12.6. Triton: 3.2.0
\        /    Bfloat16 = FALSE. FA [Xformers = 0.0.30. FA2 = False]
 "-____-"     Free license: http://github.com/unslothai/unsloth
Unsloth: Fast downloading is enabled - ignore downloading bars which are red colored!
==((====))==  Unsloth 2026.4.8: Fast Qwen2 patching. Transformers: 4.56.2. vLLM: 0.9.2.
   \\   /|    Tesla T4. Num GPUs = 2. Max memory: 14.563 GB. Platform: Linux.
O^O/ \_/ \    Torch: 2.7.0+cu126. CUDA: 7.5. CUDA Toolkit: 12.6. Triton: 3.2.0
\        /    Bfloat16 = FALSE. FA [Xformers = 0.0.30. FA2 = False]
 "-____-"     Free license: http://github.com/unslothai/unsloth
Unsloth: Fast downloading is enabled - ignore downloading bars which are red colored!
unsloth/qwen2.5-0.5b-instruct-unsloth-bnb-4bit does not have a padding token! Will use pad_token = <|PAD_TOKEN|>.
==((====))==  Unsloth 2026.4.8: Fast Qwen2 patching. Transformers: 4.56.2. vLLM: 0.9.2.
   \\   /|    Tesla T4. Num GPUs = 2. Max memory: 14.563 GB. Platform: Linux.
O^O/ \_/ \    Torch: 2.7.0+cu126. CUDA: 7.5. CUDA Toolkit: 12.6. Triton: 3.2.0
\        /    Bfloat16 = FALSE. FA [Xformers = 0.0.30. FA2 = False]
 "-____-"     Free license: http://github.com/unslothai/unsloth
Unsloth: Fast downloading is enabled - ignore downloading bars which are red colored!

Component	Detail
Environment	Local (self-contained)
Training	This notebook
Algorithm	GRPO (Group Relative Policy Optimization)
Framework	Unsloth + HF TRL

AI Incident Response — GRPO Training¶

1. Install Dependencies¶

2. Environment Setup (Self-Contained — No HF Spaces Needed)¶

3. Load Training Model with Unsloth¶

4. Reward Function + Seed Dataset¶

5. GRPO Training¶

6. Training Reward & Loss Curves¶

8. Comparison plots (removed for now)¶

9. Before/After Behavioral Demo (Theory of Mind)¶

10. Eval (fixed set + duels)¶

	mean_reward	std_reward	max_reward	format_valid_rate
attacker_baseline	0.136	0.538799	1.0	1.00
attacker_trained	0.480	0.620967	1.0	0.98
defender_baseline	-0.840	0.542586	1.0	0.68
defender_trained	-0.520	0.854166	1.0	1.00

Step	reward	reward_std	completions / mean_length	completions / min_length	completions / max_length	completions / clipped_ratio	kl	rewards / attacker_reward_func / mean	rewards / attacker_reward_func / std
1	-0.750000	0.500000	32.000000	32.000000	32.000000	1.000000	0.000019	-0.750000	0.707107
2	-0.900000	0.200000	32.000000	32.000000	32.000000	1.000000	0.000022	-0.900000	0.282843
3	-0.750000	0.500000	32.000000	32.000000	32.000000	1.000000	0.000027	-0.750000	0.707107
4	-0.900000	0.200000	32.000000	32.000000	32.000000	1.000000	0.000028	-0.900000	0.282843
5	-1.000000	0.000000	32.000000	32.000000	32.000000	1.000000	0.000022	-1.000000	0.000000
6	-1.000000	0.000000	32.000000	32.000000	32.000000	1.000000	0.000022	-1.000000	0.000000
7	-0.900000	0.200000	32.000000	32.000000	32.000000	1.000000	0.000032	-0.900000	0.282843
8	-0.600000	0.400000	32.000000	32.000000	32.000000	1.000000	0.000023	-0.600000	0.427618
9	-0.750000	0.500000	32.000000	32.000000	32.000000	1.000000	0.000027	-0.750000	0.707107
10	-0.650000	0.700000	32.000000	32.000000	32.000000	1.000000	0.000023	-0.650000	0.723089
11	-1.000000	0.000000	32.000000	32.000000	32.000000	1.000000	0.000025	-1.000000	0.000000
12	-1.000000	0.000000	32.000000	32.000000	32.000000	1.000000	0.000029	-1.000000	0.000000
13	-0.912500	0.175000	32.000000	32.000000	32.000000	1.000000	0.000030	-0.912500	0.247487
14	-0.900000	0.200000	32.000000	32.000000	32.000000	1.000000	0.000026	-0.900000	0.282843
15	-1.000000	0.000000	32.000000	32.000000	32.000000	1.000000	0.000029	-1.000000	0.000000
16	-0.750000	0.500000	32.000000	32.000000	32.000000	1.000000	0.000025	-0.750000	0.707107
17	-0.400000	0.489898	32.000000	32.000000	32.000000	1.000000	0.000032	-0.400000	0.907115
18	-0.750000	0.500000	32.000000	32.000000	32.000000	1.000000	0.000026	-0.750000	0.707107
19	-0.500000	0.577350	32.000000	32.000000	32.000000	1.000000	0.000027	-0.500000	0.925820
20	-0.250000	0.500000	32.000000	32.000000	32.000000	1.000000	0.000037	-0.250000	1.035098
21	-0.550000	0.672582	32.000000	32.000000	32.000000	1.000000	0.000032	-0.550000	0.723089
22	-0.750000	0.500000	32.000000	32.000000	32.000000	1.000000	0.000033	-0.750000	0.707107
23	-1.000000	0.000000	32.000000	32.000000	32.000000	1.000000	0.000026	-1.000000	0.000000
24	-0.500000	1.000000	32.000000	32.000000	32.000000	1.000000	0.000038	-0.500000	0.925820
25	-0.750000	0.500000	32.000000	32.000000	32.000000	1.000000	0.000039	-0.750000	0.707107
26	-1.000000	0.000000	32.000000	32.000000	32.000000	1.000000	0.000031	-1.000000	0.000000
27	-0.750000	0.500000	32.000000	32.000000	32.000000	1.000000	0.000031	-0.750000	0.707107
28	-0.662500	0.471478	32.000000	32.000000	32.000000	1.000000	0.000027	-0.662500	0.715017
29	-0.750000	0.500000	32.000000	32.000000	32.000000	1.000000	0.000042	-0.750000	0.707107
30	-0.650000	0.700000	32.000000	32.000000	32.000000	1.000000	0.000035	-0.650000	0.723089
31	-0.150000	1.049932	32.000000	32.000000	32.000000	1.000000	0.000128	-0.150000	0.989950
32	-1.000000	0.000000	32.000000	32.000000	32.000000	1.000000	0.000054	-1.000000	0.000000
33	-1.000000	0.000000	32.000000	32.000000	32.000000	1.000000	0.000040	-1.000000	0.000000
34	-1.000000	0.000000	32.000000	32.000000	32.000000	1.000000	0.000064	-1.000000	0.000000
35	-0.750000	0.500000	32.000000	32.000000	32.000000	1.000000	0.000062	-0.750000	0.707107
36	-0.900000	0.200000	32.000000	32.000000	32.000000	1.000000	0.000065	-0.900000	0.282843
37	-0.450000	0.612311	32.000000	32.000000	32.000000	1.000000	0.000120	-0.450000	0.707107
38	-0.750000	0.500000	32.000000	32.000000	32.000000	1.000000	0.000097	-0.750000	0.707107
39	-0.750000	0.500000	32.000000	32.000000	32.000000	1.000000	0.000123	-0.750000	0.707107
40	-0.750000	0.500000	32.000000	32.000000	32.000000	1.000000	0.000094	-0.750000	0.707107

Step	reward	reward_std	completions / mean_length	completions / min_length	completions / max_length	completions / clipped_ratio	kl	rewards / defender_reward_func / mean	rewards / defender_reward_func / std
1	-1.000000	0.000000	32.000000	32.000000	32.000000	1.000000	0.000022	-1.000000	0.000000
2	-1.000000	0.000000	32.000000	32.000000	32.000000	1.000000	0.000023	-1.000000	0.000000
3	-1.000000	0.000000	32.000000	32.000000	32.000000	1.000000	0.000026	-1.000000	0.000000
4	-1.000000	0.000000	32.000000	32.000000	32.000000	1.000000	0.000022	-1.000000	0.000000
5	-1.000000	0.000000	32.000000	32.000000	32.000000	1.000000	0.000023	-1.000000	0.000000
6	-0.690000	0.620000	32.000000	32.000000	32.000000	1.000000	0.000018	-0.690000	0.876812
7	-1.000000	0.000000	32.000000	32.000000	32.000000	1.000000	0.000026	-1.000000	0.000000
8	-0.650000	0.700000	32.000000	32.000000	32.000000	1.000000	0.000018	-0.650000	0.807111
9	-0.892500	0.125797	32.000000	32.000000	32.000000	1.000000	0.000020	-0.892500	0.200838
10	-0.937500	0.125000	32.000000	32.000000	32.000000	1.000000	0.000020	-0.937500	0.176777
11	-0.937500	0.125000	32.000000	32.000000	32.000000	1.000000	0.000018	-0.937500	0.176777
12	-0.825000	0.350000	32.000000	32.000000	32.000000	1.000000	0.000019	-0.825000	0.494975
13	-1.000000	0.000000	32.000000	32.000000	32.000000	1.000000	0.000025	-1.000000	0.000000
14	-0.712500	0.575000	32.000000	32.000000	32.000000	1.000000	0.000024	-0.712500	0.813173
15	-0.800000	0.294338	32.000000	32.000000	32.000000	1.000000	0.000021	-0.800000	0.277746
16	-1.000000	0.000000	32.000000	32.000000	32.000000	1.000000	0.000020	-1.000000	0.000000
17	-1.000000	0.000000	32.000000	32.000000	32.000000	1.000000	0.000021	-1.000000	0.000000
18	-0.525000	0.630594	32.000000	32.000000	32.000000	1.000000	0.000017	-0.525000	0.777817
19	-0.862500	0.160078	32.000000	32.000000	32.000000	1.000000	0.000019	-0.862500	0.255999
20	-0.712500	0.575000	32.000000	32.000000	32.000000	1.000000	0.000021	-0.712500	0.813173
21	-0.712500	0.575000	32.000000	32.000000	32.000000	1.000000	0.000030	-0.712500	0.813173
22	-1.000000	0.000000	32.000000	32.000000	32.000000	1.000000	0.000023	-1.000000	0.000000
23	-0.943125	0.113750	32.000000	32.000000	32.000000	1.000000	0.000023	-0.943125	0.160867
24	-0.884375	0.231250	32.000000	32.000000	32.000000	1.000000	0.000027	-0.884375	0.215032
25	-0.937500	0.125000	32.000000	32.000000	32.000000	1.000000	0.000031	-0.937500	0.176777
26	-0.937500	0.125000	32.000000	32.000000	32.000000	1.000000	0.000031	-0.937500	0.176777
27	-1.000000	0.000000	32.000000	32.000000	32.000000	1.000000	0.000031	-1.000000	0.000000
28	-0.937500	0.125000	32.000000	32.000000	32.000000	1.000000	0.000035	-0.937500	0.176777
29	-0.778125	0.231250	32.000000	32.000000	32.000000	1.000000	0.000028	-0.778125	0.238461
30	-0.637500	0.543714	32.000000	32.000000	32.000000	1.000000	0.000022	-0.637500	0.810533
31	-0.890000	0.220000	32.000000	32.000000	32.000000	1.000000	0.000050	-0.890000	0.206190
32	-0.937500	0.125000	32.000000	32.000000	32.000000	1.000000	0.000041	-0.937500	0.176777
33	-1.000000	0.000000	32.000000	32.000000	32.000000	1.000000	0.000038	-1.000000	0.000000
34	-0.650000	0.700000	32.000000	32.000000	32.000000	1.000000	0.000039	-0.650000	0.807111
35	-1.000000	0.000000	32.000000	32.000000	32.000000	1.000000	0.000054	-1.000000	0.000000
36	-0.925000	0.150000	32.000000	32.000000	32.000000	1.000000	0.000083	-0.925000	0.212132
37	-1.000000	0.000000	32.000000	32.000000	32.000000	1.000000	0.000050	-1.000000	0.000000
38	-0.862500	0.275000	32.000000	32.000000	32.000000	1.000000	0.000075	-0.862500	0.255999
39	-0.937500	0.125000	32.000000	32.000000	32.000000	1.000000	0.000109	-0.937500	0.176777
40	-0.952500	0.095000	32.000000	32.000000	32.000000	1.000000	0.000056	-0.952500	0.134350