Removed training loops below render function

cloud_arena_final.py

@@ -943,659 +943,3 @@ class CloudArenaEnv(gym.Env):
     def render(self): pass
 
 
-# ── Gymnasium wrapper ─────────────────────────────────────────────────────────
-
-from sb3_contrib.common.wrappers import ActionMasker
-
-def get_action_masks(env) -> np.ndarray:
-    """Extract mask through ActionMasker wrapper."""
-    inner = env
-    while hasattr(inner, "env"):
-        inner = inner.env
-    return inner.action_masks()
-
-
-# ── Sanity checks ─────────────────────────────────────────────────────────────
-
-print("=" * 60)
-print("  CLOUD ARENA — ENVIRONMENT SANITY CHECKS")
-print("=" * 60)
-print(f"  OBS_DIM   = {OBS_DIM}")
-print(f"  N_ACTIONS = {N_ACTIONS}")
-
-_cr = [0]; _gr = [0]
-_e = ActionMasker(CloudArenaEnv(_cr, _gr), get_action_masks)
-_obs, _ = _e.reset()
-assert _obs.shape == (OBS_DIM,), f"Bad obs shape: {_obs.shape}"
-print(f"\n  ✅ Obs shape     : {_obs.shape}")
-
-_mask = _e.env.action_masks()
-assert _mask.shape == (N_ACTIONS,), f"Bad mask: {_mask.shape}"
-print(f"  ✅ Mask shape    : {_mask.shape}, {_mask.sum()} valid actions")
-
-# NOOP must be penalized when system is degraded
-_cr[0] = 0
-_e2 = ActionMasker(CloudArenaEnv(_cr, [0]), get_action_masks)
-_e2.reset()
-# Manually degrade a resource to force NOOP penalty
-_e2.env.resources[1].risk_score = 0.8
-_, r_noop, _, _, _ = _e2.step(A_NOOP * MAX_RESOURCES)
-print(f"  ✅ NOOP (degraded): reward={r_noop:.3f} (should be negative)")
-
-# Veto: RESIZE_DOWN on a non-overprovisioned resource
-_e3 = ActionMasker(CloudArenaEnv([0], [0]), get_action_masks)
-_e3.reset()
-_e3.env.resources[0].allocated = 0.30
-_e3.env.resources[0].usage     = 0.28   # not overprovisioned
-_, r_v, _, _, i_v = _e3.step(A_RESIZE_DOWN * MAX_RESOURCES)
-print(f"  ✅ Veto test     : veto={i_v.get('veto_rate', '?'):.2f}, r={r_v:.3f}")
-
-# Phase 0 win condition reachability
-_e4 = ActionMasker(CloudArenaEnv([0], [0]), get_action_masks)
-_e4.reset()
-ic = _e4.env.initial_total_cost
-wt = WIN_COST_THR[0]
-print(f"\n  Phase 0 win target: cost < {ic * wt:.3f} (initial={ic:.3f})")
-print(f"  Threshold is {wt*100:.0f}% of initial — requires ~{(1-wt)*100:.0f}% cost reduction")
-
-print("\n  8-step walkthrough:")
-_e5 = ActionMasker(CloudArenaEnv([0], [0]), get_action_masks)
-_e5.reset()
-_actions_to_test = [
-    A_ANALYZE * MAX_RESOURCES,        # analyze resource 0
-    A_ANALYZE * MAX_RESOURCES + 1,    # analyze resource 1
-    A_RESIZE_DOWN * MAX_RESOURCES + 1,# resize down resource 1
-    A_RESIZE_DOWN * MAX_RESOURCES + 2,# resize down resource 2
-    A_STOP * MAX_RESOURCES + 3,       # stop resource 3 (LOW crit)
-    A_NOOP * MAX_RESOURCES,           # noop
-    A_RESIZE_DOWN * MAX_RESOURCES + 1,# resize down again (should veto)
-    A_NOOP * MAX_RESOURCES,           # noop
-]
-for i, a in enumerate(_actions_to_test):
-    _, r, t, tr, inf = _e5.step(a)
-    atype = a // MAX_RESOURCES
-    ridx  = a % MAX_RESOURCES
-    print(f"    {i+1}. {['NOOP','ANALYZE','VRF_DEP','RESIZE_DN','RESIZE_UP','STOP','RESTART','DELETE','PATCH','ENCRYPT','RESTRICT','ROT_CRED','EN_LOG','ARCHIVE','OPT_NET'][atype]:<12}"
-          f" r{ridx}  reward={r:+.3f}  win={inf.get('win',0)}  savings={inf.get('savings_pct',0):.1f}%")
-
-print("\n  ✅ All sanity checks passed — proceed to Cell 3.\n")
-
-# ══════════════════════════════════════════════════════════════════════════════
-#  CELL 3 — TRAINING (FINAL FIXED VERSION)
-# ══════════════════════════════════════════════════════════════════════════════
-
-import os, torch, math, sys
-import numpy as np
-import matplotlib.pyplot as plt
-
-from stable_baselines3.common.vec_env import DummyVecEnv, VecNormalize, sync_envs_normalization
-from stable_baselines3.common.callbacks import BaseCallback
-from stable_baselines3.common.monitor import Monitor
-
-from sb3_contrib import MaskablePPO
-from sb3_contrib.common.maskable.callbacks import MaskableEvalCallback
-
-torch.manual_seed(GLOBAL_SEED)
-
-for d in ["./logs/", "./eval_logs/", "./models/"]:
-    os.makedirs(d, exist_ok=True)
-
-TOTAL_TIMESTEPS = 500_000
-_curriculum_ref  = [0]
-_global_step_ref = [0]
-
-# ================================
-# LR SCHEDULE
-# ================================
-def cosine_lr(progress_remaining: float, init_lr: float = 3e-4, min_lr: float = 5e-5):
-    return min_lr + (init_lr - min_lr) * 0.5 * (1.0 + math.cos(math.pi * (1.0 - progress_remaining)))
-
-# ================================
-# SAFE EVAL CALLBACK (FIXED)
-# ================================
-class SafeMaskableEvalCallback(MaskableEvalCallback):
-    def _on_step(self) -> bool:
-        if self.model.get_vec_normalize_env() is not None:
-            sync_envs_normalization(self.training_env, self.eval_env)
-        return super()._on_step()
-
-# ================================
-# CALLBACK (FIXED UI MATH)
-# ================================
-class CloudArenaCallback(BaseCallback):
-    EMA_ALPHA = 0.02
-    MIN_EPS_PER_PHASE = 800
-    PHASE_THRESHOLDS = {0: 0.65, 1: 0.62, 2: 0.58, 3: 0.55, 4: 0.52}
-    PROGRESS_EVERY = 500
-
-    def __init__(self, verbose=0):
-        super().__init__(verbose)
-        self.ema_win_rate = 0.0
-        self.current_level = 0
-        self._phase_eps = 0
-        # Added for dashboard support
-        self.episode_rewards = []
-        self.episode_wins = []
-        self.episode_savings = []
-        self.episode_security = []
-        self.episode_veto_rates = []
-        self.curriculum_log = [(0, 0)]
-        self.action_freq = np.zeros(N_ACTION_TYPES)
-
-    def _on_step(self) -> bool:
-        if self.num_timesteps % self.PROGRESS_EVERY == 0:
-            self._print_progress()
-
-        # Log actions
-        actions = self.locals.get("actions")
-        if actions is not None:
-            for a in actions:
-                atype = int(a) // MAX_RESOURCES
-                if atype < N_ACTION_TYPES: self.action_freq[atype] += 1
-
-        dones = self.locals.get("dones", [False])
-        if dones[0]:
-            info = self.locals.get("infos", [{}])[0]
-            self._on_episode_end(info)
-        return True
-
-    def _on_episode_end(self, info: dict):
-        if "final_info" in info: info = info["final_info"]
-        win = int(info.get("win", 0))
-        self.ema_win_rate = ((1 - self.EMA_ALPHA) * self.ema_win_rate + self.EMA_ALPHA * win)
-
-        # Logging for Dashboard
-        self.episode_rewards.append(float(self.locals.get("rewards", [0])[0]))
-        self.episode_wins.append(win)
-        self.episode_savings.append(info.get("savings_pct", 0))
-        self.episode_security.append(info.get("security_score", 0))
-        self.episode_veto_rates.append(info.get("veto_rate", 0))
-
-        self._phase_eps += 1
-        thr = self.PHASE_THRESHOLDS.get(self.current_level, 0.50)
-        if self.current_level < 5 and self._phase_eps >= self.MIN_EPS_PER_PHASE and self.ema_win_rate >= thr:
-            self._try_promote()
-
-    def _try_promote(self):
-        self.current_level += 1
-        _curriculum_ref[0] = self.current_level
-        self._phase_eps = 0
-        self.ema_win_rate = 0.0
-        self.curriculum_log.append((self.num_timesteps, self.current_level))
-        print(f"\n✄ PROMOTED -> Phase {self.current_level}")
-
-    def _print_progress(self):
-        pct = min(100.0, self.num_timesteps / TOTAL_TIMESTEPS * 100)
-        sys.stdout.write(f"\rProgress: {pct:.1f}% | Phase: {self.current_level} | EMA Win: {self.ema_win_rate*100:.1f}%")
-        sys.stdout.flush()
-
-# ================================
-# ENV SETUP
-# ================================
-def make_env():
-    env = CloudArenaEnv(_curriculum_ref, _global_step_ref)
-    env = Monitor(env)
-    return ActionMasker(env, get_action_masks)
-
-train_env = DummyVecEnv([make_env])
-train_env = VecNormalize(train_env, norm_obs=True, norm_reward=True, clip_obs=10.0)
-
-eval_env = DummyVecEnv([make_env])
-eval_env = VecNormalize(eval_env, norm_obs=True, norm_reward=False, training=False)
-eval_env.obs_rms = train_env.obs_rms
-
-# ================================
-# MODEL & CALLBACK INIT
-# ================================
-model = MaskablePPO("MlpPolicy", train_env, learning_rate=cosine_lr, ent_coef=0.01, verbose=0)
-arena_cb = CloudArenaCallback()
-eval_cb = SafeMaskableEvalCallback(eval_env, best_model_save_path="./models/", eval_freq=10000)
-
-# ================================
-# TRAIN
-# ================================
-print("Starting Pipeline...")
-model.learn(total_timesteps=TOTAL_TIMESTEPS, callback=[arena_cb, eval_cb])
-
-# SAVE FINAL ARTIFACTS
-model.save("./models/cloud_arena_final")
-train_env.save("./models/cloud_arena_vecnorm.pkl")
-print("\n✅ Model and VecNormalize stats saved.")
-
-# ══════════════════════════════════════════════════════════════════════════════
-#  CELL 4 — EVALUATION (30 deterministic episodes)
-# ══════════════════════════════════════════════════════════════════════════════
-
-import numpy as np
-from stable_baselines3.common.vec_env import DummyVecEnv, VecNormalize
-from sb3_contrib import MaskablePPO
-
-print("\n" + "=" * 60)
-print("  EVALUATION — 30 Deterministic Episodes")
-print("=" * 60)
-
-eval_level = arena_cb.current_level
-
-def make_eval_env2(level: int, seed: int = 9999):
-    def _init():
-        env = CloudArenaEnv(
-            curriculum_ref=[level],
-            global_step_ref=[TOTAL_TIMESTEPS])
-        return ActionMasker(env, get_action_masks)
-    return _init
-
-# Ensure artifacts exist before loading
-if not os.path.exists("./models/cloud_arena_vecnorm.pkl"):
-    print("❌ Error: vecnorm.pkl not found. Did you finish training in Cell 3?")
-else:
-    raw_eval2 = DummyVecEnv([make_eval_env2(eval_level)])
-    eval_env2 = VecNormalize.load("./models/cloud_arena_vecnorm.pkl", raw_eval2)
-    eval_env2.training    = False
-    eval_env2.norm_reward = False
-
-    eval_model = MaskablePPO.load("./models/cloud_arena_final", env=eval_env2)
-
-    def _get_inner(vec_env):
-        inner = vec_env.envs[0]
-        while hasattr(inner, "env"):
-            inner = inner.env
-        return inner
-
-    N_EVAL = 30
-    results = {k: [] for k in ["win","cost_score","security_score","reliability_score",
-                                "savings_pct","veto_rate","cascade_count","steps"]}
-
-    for ep in range(N_EVAL):
-        obs = eval_env2.reset()
-        done = False; steps = 0
-        while not done:
-            masks = [_get_inner(eval_env2).action_masks()]
-            act, _ = eval_model.predict(obs, deterministic=True, action_masks=masks)
-            obs, rew, done_arr, info_arr = eval_env2.step(act)
-            done = bool(done_arr[0]); steps += 1
-        info = info_arr[0] if info_arr else {}
-        for k in results:
-            results[k].append(info.get(k, 0) if k != "steps" else steps)
-
-        print(f"  Ep {ep+1:>2}: {'WIN ' if info.get('win') else 'LOSS'} | "
-              f"reward N/A | "
-              f"cost={info.get('cost_score',0):.2f} | "
-              f"sec={info.get('security_score',0):.2f} | "
-              f"sav={info.get('savings_pct',0):.1f}% | "
-              f"steps={steps}")
-
-    wr      = np.mean(results["win"])  * 100
-    cost_sc = np.mean(results["cost_score"])
-    sec_sc  = np.mean(results["security_score"])
-    rel_sc  = np.mean(results["reliability_score"])
-    savings = np.mean(results["savings_pct"])
-    veto_r  = np.mean(results["veto_rate"]) * 100
-    casc_r  = np.mean([c > 0 for c in results["cascade_count"]]) * 100
-    avg_s   = np.mean(results["steps"])
-
-    composite = (wr + cost_sc * 100 + sec_sc * 100 + rel_sc * 100) / 4
-    tier = ("🥇 PLATINUM" if composite >= 90 else
-            "🥈 GOLD"     if composite >= 75 else
-            "🥉 SILVER"   if composite >= 60 else
-            "🔶 BRONZE"   if composite >= 45 else
-            "❌ FAILING")
-
-    print(f"\n{'='*40}")
-    print(f"  EVALUATION RESULTS ({N_EVAL} eps | Phase={eval_level})")
-    print(f"{'='*40}")
-    print(f"  Win Rate       : {wr:.1f}%")
-    print(f"  Cost Score     : {cost_sc:.3f}")
-    print(f"  Security Score : {sec_sc:.3f}")
-    print(f"  Reliability    : {rel_sc:.3f}")
-    print(f"  Avg Savings    : {savings:.1f}%")
-    print(f"  Veto Rate      : {veto_r:.1f}%")
-    print(f"  Cascade Rate   : {casc_r:.1f}%")
-    print(f"  Avg Steps      : {avg_s:.1f}")
-    print(f"  Composite      : {composite:.1f}%")
-    print(f"{'='*40}")
-    print(f"  TIER: {tier}")
-    print(f"{'='*40}")
-
-# ══════════════════════════════════════════════════════════════════════════════
-#  CELL 5 — BOSS FIGHTS
-# ══════════════════════════════════════════════════════════════════════════════
-
-import os, torch
-import numpy as np
-from typing import List
-from stable_baselines3.common.vec_env import DummyVecEnv, VecNormalize
-from sb3_contrib import MaskablePPO
-
-print("\n" + "=" * 60)
-print("  BOSS FIGHTS")
-print("=" * 60)
-
-BOSS_NAMES = {
-    1: "Cost Crisis",
-    2: "Security Breach",
-    3: "Infrastructure Failure (NOOP Test)",
-    4: "Traffic Surge",
-    5: "Final Boss (Multi-Failure)",
-}
-
-def _get_inner(vec_env):
-    inner = vec_env.envs[0]
-    while hasattr(inner, "env"):
-        inner = inner.env
-    return inner
-
-def run_boss_fight(model, level: int, scenario_id: int, n_runs: int = 10) -> List[dict]:
-    results = []
-    for seed in range(100, 100 + n_runs):
-        def _init():
-            env = CloudArenaEnv(curriculum_ref=[level], global_step_ref=[0])
-            return ActionMasker(env, get_action_masks)
-
-        raw = DummyVecEnv([_init])
-        if not os.path.exists("./models/cloud_arena_vecnorm.pkl"):
-            raise FileNotFoundError("Missing vecnorm.pkl artifacts.")
-
-        vec = VecNormalize.load("./models/cloud_arena_vecnorm.pkl", raw)
-        vec.training = False; vec.norm_reward = False
-
-        inner = _get_inner(vec)
-        # Reset the specific scenario
-        raw_obs, _ = inner.reset(seed=seed, options={"scenario": scenario_id})
-        obs = vec.normalize_obs(np.array([raw_obs]))
-
-        done = False; steps = 0
-        noops_chaos = 0; chaos_steps_total = 0
-
-        while not done:
-            masks = [inner.action_masks()]
-            act, _ = model.predict(obs, deterministic=True, action_masks=masks)
-            a_type = int(act[0]) // MAX_RESOURCES
-
-            if inner.chaos_active:
-                chaos_steps_total += 1
-                if a_type == A_NOOP: noops_chaos += 1
-
-            obs, _, done_arr, info_arr = vec.step(act)
-            done = bool(done_arr[0]); steps += 1
-
-        info = info_arr[0] if info_arr else {}
-        info.update({"steps": steps, "noops_chaos": noops_chaos, "chaos_steps": chaos_steps_total})
-        results.append(info)
-        vec.close()
-    return results
-
-boss_scores = {}
-overall_boss = 0.0
-# Load model if not in memory
-try:
-    eval_model = MaskablePPO.load("./models/cloud_arena_final")
-    eval_level = _curriculum_ref[0]
-except:
-    print("⚠️ Could not auto-load model. Ensure Cell 3 finished.")
-
-for s_id, name in BOSS_NAMES.items():
-    print(f"\n▶ Boss Fight {s_id}: {name}")
-    try:
-        runs = run_boss_fight(eval_model, eval_level, s_id, n_runs=10)
-        wins = [r.get("win", 0) for r in runs]
-        costs = [r.get("cost_score", 0) for r in runs]
-        secs = [r.get("security_score", 0) for r in runs]
-        rels = [r.get("reliability_score", 0) for r in runs]
-        saves = [r.get("savings_pct", 0) for r in runs]
-        casc = [r.get("cascade_count", 0) for r in runs]
-        steps = [r.get("steps", MAX_STEPS) for r in runs]
-
-        # Scoring Logic
-        if s_id == 3: # NOOP Test
-            noop_r = [r["noops_chaos"] / max(r["chaos_steps"], 1) for r in runs]
-            score = (0.4*np.mean(noop_r) + 0.6*np.mean(rels)) * 100
-        else:
-            score = (0.4*np.mean(wins) + 0.3*np.mean(costs) + 0.3*np.mean(secs)) * 100
-
-        boss_scores[s_id] = score
-        print(f"   Score: {score:.1f}% | Win Rate: {np.mean(wins)*100:.0f}%")
-    except Exception as ex:
-        print(f"   ⚠️ Error: {ex}")
-
-if boss_scores:
-    overall_boss = float(np.mean(list(boss_scores.values())))
-    otier = ("🥇 PLATINUM" if overall_boss >= 90 else "🥈 GOLD" if overall_boss >= 75 else "🥉 SILVER")
-    print(f"\nFINAL BOSS SCORE: {overall_boss:.1f}% -> {otier}")
-
-# ══════════════════════════════════════════════════════════════════════════════
-#  CELL 6 — TRAINING DASHBOARD (9 panels)
-# ══════════════════════════════════════════════════════════════════════════════
-
-import matplotlib
-matplotlib.use("Agg")
-import matplotlib.pyplot as plt
-import matplotlib.gridspec as gridspec
-import numpy as np
-import warnings
-warnings.filterwarnings("ignore")
-
-print("\n📊 Generating training dashboard...")
-
-# Standardized Colors
-BG, PANEL, GRID, TEXT = "#000000", "#0a0f1a", "#1a2030", "#e0e8ff"
-C_REWARD, C_WIN, C_SAVINGS, C_SEC, C_PHASE = "#1e5f8c", "#2ecc71", "#f39c12", "#00cca3", "#7a5fff"
-
-# Data Prep
-rw = np.array(arena_cb.episode_rewards)
-wn = np.array(arena_cb.episode_wins) * 100
-sv = np.array(arena_cb.episode_savings)
-sec = np.array(arena_cb.episode_security)
-ep = np.arange(len(rw))
-
-fig = plt.figure(figsize=(20, 14), facecolor=BG)
-gs = gridspec.GridSpec(3, 3, figure=fig, hspace=0.4, wspace=0.3)
-
-# Helper for smoothing
-def smooth(y, box_pts=50):
-    if len(y) < box_pts: return y
-    box = np.ones(box_pts)/box_pts
-    return np.convolve(y, box, mode='valid')
-
-# --- Panel 1: Rewards ---
-ax1 = fig.add_subplot(gs[0, 0])
-ax1.plot(rw, color=C_REWARD, alpha=0.3)
-ax1.plot(smooth(rw), color="#4a90d9", lw=2)
-ax1.set_title("Episode Rewards", color=TEXT)
-
-# --- Panel 2: Win Rate ---
-ax2 = fig.add_subplot(gs[0, 1])
-ax2.plot(smooth(wn), color=C_WIN, lw=2)
-ax2.set_ylim(0, 105)
-ax2.set_title("Rolling Win Rate (%)", color=TEXT)
-
-# --- Panel 5: Security ---
-ax5 = fig.add_subplot(gs[1, 1])
-ax5.plot(smooth(sec), color=C_SEC, lw=2)
-ax5.set_ylim(0, 1)
-ax5.set_title("Security Score", color=TEXT)
-
-# --- Panel 6: Summary Table ---
-ax6 = fig.add_subplot(gs[1, 2])
-ax6.axis('off')
-summary_text = (
-    f"TRAINING SUMMARY\n"
-    f"------------------\n"
-    f"Episodes: {len(rw)}\n"
-    f"Final Phase: {arena_cb.current_level}\n"
-    f"EMA Win Rate: {arena_cb.ema_win_rate*100:.1f}%\n"
-    f"Avg Savings: {np.mean(sv):.1f}%\n"
-    f"Boss Score: {overall_boss:.1f}%\n"
-    f"FINAL TIER: {otier if 'otier' in locals() else 'N/A'}"
-)
-ax6.text(0.1, 0.5, summary_text, color=TEXT, fontsize=12, family='monospace', va='center')
-
-# --- Panel 7: Curriculum ---
-ax7 = fig.add_subplot(gs[2, 0])
-if hasattr(arena_cb, 'curriculum_log'):
-    steps, lvls = zip(*arena_cb.curriculum_log)
-    ax7.step(steps, lvls, where='post', color=C_PHASE, lw=2)
-ax7.set_title("Curriculum Progression", color=TEXT)
-
-# --- Panel 8: Boss Scores ---
-ax8 = fig.add_subplot(gs[2, 1])
-if 'boss_scores' in locals() and boss_scores:
-    ax8.bar(BOSS_NAMES.values(), [boss_scores.get(i, 0) for i in BOSS_NAMES], color=C_PHASE)
-    ax8.set_xticklabels(BOSS_NAMES.values(), rotation=45, ha='right', fontsize=8)
-ax8.set_title("Boss Fight Performance", color=TEXT)
-
-plt.tight_layout()
-plt.savefig("training_dashboard_full.png", dpi=150, facecolor=BG)
-plt.show()
-print("✅ Dashboard generated: training_dashboard_full.png")
-
-import matplotlib.pyplot as plt
-import numpy as np
-from google.colab import files
-
-# --- Reference Image Color Palette ---
-REF_BG = '#0e1117' # Dark navy/grey background
-REF_CYAN = '#00d4ff' # Reward line
-REF_AMBER = '#ffa500' # Savings line
-REF_NEON = '#39ff14' # Security line
-TEXT_COLOR = '#e6e6e6'
-
-rewards = np.array(arena_cb.episode_rewards)
-savings = np.array(arena_cb.episode_savings)
-security = np.array(arena_cb.episode_security)
-
-def smooth_data(data, window=100):
-    if len(data) < window: return data
-    return np.convolve(data, np.ones(window)/window, mode='valid')
-
-# --- Plotting ---
-fig, (ax1, ax2, ax3) = plt.subplots(1, 3, figsize=(22, 6), facecolor=REF_BG)
-
-for ax in [ax1, ax2, ax3]:
-    ax.set_facecolor(REF_BG)
-    ax.grid(True, alpha=0.05, color='white', linestyle='-')
-    ax.spines['top'].set_visible(False)
-    ax.spines['right'].set_visible(False)
-    ax.spines['left'].set_color('#333333')
-    ax.spines['bottom'].set_color('#333333')
-    ax.tick_params(colors=TEXT_COLOR, labelsize=10)
-
-# 1. Learning Curve (Cyan)
-ax1.plot(rewards, color=REF_CYAN, alpha=0.15)
-ax1.plot(smooth_data(rewards), color=REF_CYAN, lw=3)
-ax1.set_title("Learning Curve (Rewards)", color=TEXT_COLOR, fontsize=14, fontweight='bold', pad=20)
-ax1.set_ylabel("Total Reward", color=TEXT_COLOR, alpha=0.8)
-
-# 2. Cost Optimization (Amber)
-ax2.plot(savings, color=REF_AMBER, alpha=0.15)
-ax2.plot(smooth_data(savings), color=REF_AMBER, lw=3)
-ax2.set_title("Cloud Cost Optimization %", color=TEXT_COLOR, fontsize=14, fontweight='bold', pad=20)
-ax2.set_ylabel("Savings Pct", color=TEXT_COLOR, alpha=0.8)
-ax2.set_ylim(0, 100)
-
-# 3. Security Posture (Neon Green)
-ax3.plot(security, color=REF_NEON, alpha=0.15)
-ax3.plot(smooth_data(security), color=REF_NEON, lw=3)
-ax3.set_title("Security Posture Score", color=TEXT_COLOR, fontsize=14, fontweight='bold', pad=20)
-ax3.set_ylabel("Normalized Score", color=TEXT_COLOR, alpha=0.8)
-ax3.set_ylim(0, 1)
-
-plt.tight_layout()
-filename = 'ref_styled_report.png'
-plt.savefig(filename, dpi=200, bbox_inches='tight', facecolor=REF_BG)
-plt.show()
-
-# Trigger Download
-files.download(filename)
-
-import matplotlib.pyplot as plt
-import matplotlib.gridspec as gridspec
-import numpy as np
-from google.colab import files
-
-# --- Color Palette from Reference ---
-REF_BG = '#0e1117'
-REF_CYAN = '#00d4ff'
-REF_AMBER = '#ffa500'
-REF_NEON = '#39ff14'
-REF_PURPLE = '#bc13fe'
-TEXT_COLOR = '#e6e6e6'
-
-# --- Data Extraction ---
-rw = np.array(arena_cb.episode_rewards)
-wn = np.array(arena_cb.episode_wins) * 100
-sv = np.array(arena_cb.episode_savings)
-sec = np.array(arena_cb.episode_security)
-vt = np.array(arena_cb.episode_veto_rates) * 100
-# Reliability as 'Crash Rate' (1 - reliability score)
-cr = (1.0 - np.array(arena_cb.episode_security)) * 100
-# Action frequencies
-act_freq = arena_cb.action_freq
-act_labels = ['NOOP','ANALYZE','VRF_DEP','RESIZE_DN','RESIZE_UP','STOP','RESTART','DELETE','PATCH','ENCRYPT','RESTRICT','ROT_CRED','EN_LOG','ARCHIVE','OPT_NET']
-
-def smooth(y, box_pts=50):
-    if len(y) < box_pts: return y
-    return np.convolve(y, np.ones(box_pts)/box_pts, mode='valid')
-
-# --- Layout ---
-fig = plt.figure(figsize=(24, 18), facecolor=REF_BG)
-gs = gridspec.GridSpec(3, 3, figure=fig, hspace=0.4, wspace=0.3)
-
-def style_ax(ax, title):
-    ax.set_facecolor(REF_BG)
-    ax.set_title(title, color=TEXT_COLOR, fontsize=14, fontweight='bold', pad=15)
-    ax.tick_params(colors=TEXT_COLOR, labelsize=10)
-    ax.grid(True, alpha=0.05, color='white')
-    for s in ['top','right']: ax.spines[s].set_visible(False)
-    for s in ['left','bottom']: ax.spines[s].set_color('#333333')
-
-# 1. Episode Reward
-ax1 = fig.add_subplot(gs[0, 0]); style_ax(ax1, "Episode Reward")
-ax1.plot(rw, color=REF_CYAN, alpha=0.2)
-ax1.plot(smooth(rw), color=REF_CYAN, lw=2)
-
-# 2. Rolling Win Rate
-ax2 = fig.add_subplot(gs[0, 1]); style_ax(ax2, "Rolling Win Rate (%)")
-ax2.plot(smooth(wn), color=REF_NEON, lw=2)
-ax2.set_ylim(0, 105)
-
-# 3. Rolling Crash Rate
-ax3 = fig.add_subplot(gs[0, 2]); style_ax(ax3, "Rolling Crash Rate (%)")
-ax3.plot(smooth(cr), color='#ff4b2b', lw=2)
-ax3.set_ylim(0, 105)
-
-# 4. Rolling Avg Savings
-ax4 = fig.add_subplot(gs[1, 0]); style_ax(ax4, "Rolling Avg Savings (%)")
-ax4.plot(smooth(sv), color=REF_AMBER, lw=2)
-ax4.set_ylim(0, 105)
-
-# 5. Guard Rail Veto Rate
-ax5 = fig.add_subplot(gs[1, 1]); style_ax(ax5, "Guard Rail Veto Rate (%)")
-ax5.plot(smooth(vt), color=REF_PURPLE, lw=2)
-ax5.set_ylim(0, max(10, np.max(smooth(vt))*1.2))
-
-# 6. Training Summary
-ax6 = fig.add_subplot(gs[1, 2])
-ax6.axis('off')
-summary_txt = (
-    f"TRAINING SUMMARY\n"
-    f"------------------\n"
-    f"Total Episodes: {len(rw)}\n"
-    f"Current Phase: {arena_cb.current_level}\n"
-    f"Final Win Rate: {wn[-1]:.1f}%\n"
-    f"Mean Savings: {np.mean(sv):.1f}%\n"
-    f"Avg Veto Rate: {np.mean(vt):.2f}%\n"
-    f"Status: {'SUCCESS' if wn[-1]>80 else 'TRAINING'}"
-)
-ax6.text(0.1, 0.5, summary_txt, color=TEXT_COLOR, fontsize=16, family='monospace', va='center')
-
-# 7. Action Distribution
-ax7 = fig.add_subplot(gs[2, :]); style_ax(ax7, "Action Execution Frequency")
-ax7.bar(act_labels, act_freq, color=REF_CYAN, alpha=0.8)
-ax7.set_xticklabels(act_labels, rotation=30, ha='right')
-
-plt.tight_layout()
-filename = 'comprehensive_rl_report.png'
-plt.savefig(filename, dpi=150, facecolor=REF_BG)
-plt.show()
-
-files.download(filename)