enhance dash styles

app/api/admin_ops.py

@@ -0,0 +1,148 @@
+from fastapi import APIRouter, Depends, HTTPException, Body
+from pydantic import BaseModel
+import logging
+
+from app.core.auth import verify_api_key
+from app.services.chat.api.llm_router import llm_router, circuit_registry
+import app.services.chat.api.llm_router as router_module
+from app.core.redis_client import redis_client
+
+logger = logging.getLogger(__name__)
+
+router = APIRouter(
+    prefix="/admin",
+    tags=["admin"],
+    dependencies=[Depends(verify_api_key)]
+)
+
+class BulkheadUpdate(BaseModel):
+    limit: int
+
+class CircuitTune(BaseModel):
+    failure_threshold: int
+    recovery_timeout: int
+
+class ModelBanReq(BaseModel):
+    model_name: str
+
+class KillSwitchReq(BaseModel):
+    active: bool
+
+
+# ─── Provider Controls ────────────────────────────────────────────────────────
+
+@router.post("/provider/{name}/disable")
+async def disable_provider(name: str):
+    """Manually force a provider offline unconditionally."""
+    state = llm_router.provider_states.get(name)
+    if not state:
+        raise HTTPException(status_code=404, detail="Provider not found")
+    state.is_permanently_disabled = True
+    logger.critical(f"Admin action: Provider {name} FORCE DISABLED.")
+    return {"status": "success", "provider": name, "disabled": True}
+
+@router.post("/provider/{name}/enable")
+async def enable_provider(name: str):
+    """Restore a manually disabled provider."""
+    state = llm_router.provider_states.get(name)
+    if not state:
+        raise HTTPException(status_code=404, detail="Provider not found")
+    state.is_permanently_disabled = False
+    state.disable_until = 0.0
+    logger.warning(f"Admin action: Provider {name} ENABLED.")
+    return {"status": "success", "provider": name, "disabled": False}
+
+@router.post("/provider/{name}/bulkhead")
+async def update_bulkhead(name: str, payload: BulkheadUpdate):
+    """Dynamically resize concurrency limit to throttle/boost a provider."""
+    ok = llm_router.set_bulkhead_limit(name, payload.limit)
+    if not ok:
+        raise HTTPException(status_code=404, detail="Provider not found")
+    return {"status": "success", "provider": name, "new_limit": payload.limit}
+
+
+# ─── Circuit Controls ─────────────────────────────────────────────────────────
+
+@router.post("/circuit/{name}/trip")
+async def trip_circuit(name: str):
+    """Force circuit OPEN (throws it into cooldown natively)."""
+    cb = circuit_registry.get(name)
+    if not cb:
+        raise HTTPException(status_code=404, detail="Circuit not found")
+    cb.record_failure(recovery_override=86400) # Open for 24h (admin tripped)
+    logger.critical(f"Admin action: Circuit for {name} TRIPPED.")
+    return {"status": "success", "provider": name, "circuit_state": cb.state.value}
+
+@router.post("/circuit/{name}/reset")
+async def reset_circuit(name: str):
+    """Force circuit CLOSED (instant heal)."""
+    cb = circuit_registry.get(name)
+    if not cb:
+        raise HTTPException(status_code=404, detail="Circuit not found")
+    cb.record_success()
+    state = llm_router.provider_states.get(name)
+    if state:
+        state.consecutive_failures = 0
+        state.cooldown_until = 0.0
+    logger.warning(f"Admin action: Circuit for {name} RESET to CLOSED.")
+    return {"status": "success", "provider": name, "circuit_state": cb.state.value}
+
+
+@router.post("/circuit/{name}/tune")
+async def tune_circuit(name: str, payload: CircuitTune):
+    """Dynamically adjust failure threshold and base recovery timeout."""
+    cb = circuit_registry.get(name)
+    if not cb:
+        raise HTTPException(status_code=404, detail="Circuit not found")
+    cb.failure_threshold = max(1, payload.failure_threshold)
+    cb._recovery_base = max(1, payload.recovery_timeout)
+    cb.recovery_timeout = cb._recovery_base
+    logger.warning(f"Admin action: Circuit {name} tuned (thr={cb.failure_threshold}, timeout={cb._recovery_base})")
+    return {"status": "success", "provider": name, "tuned": payload.dict()}
+
+
+# ─── Model & Global Controls ──────────────────────────────────────────────────
+
+@router.post("/model/ban")
+async def ban_model(payload: ModelBanReq):
+    """Add a model string to the global banlist."""
+    router_module.banned_models.add(payload.model_name)
+    logger.critical(f"Admin action: Model {payload.model_name} GLOBALLY BANNED.")
+    return {"status": "success", "banned_models": list(router_module.banned_models)}
+
+@router.post("/model/unban")
+async def unban_model(payload: ModelBanReq):
+    """Remove a model string from the global banlist."""
+    router_module.banned_models.discard(payload.model_name)
+    logger.warning(f"Admin action: Model {payload.model_name} unbanned.")
+    return {"status": "success", "banned_models": list(router_module.banned_models)}
+
+@router.post("/system/killswitch")
+async def toggle_killswitch(payload: KillSwitchReq):
+    """Global emergency stop. All LLM calls return 503 instantly."""
+    router_module.KILL_SWITCH_ACTIVE = payload.active
+    if payload.active:
+        logger.critical("🚨 ADMIN ACTION: GLOBAL KILL SWITCH ACTIVATED. System halted.")
+    else:
+        logger.warning("🟢 ADMIN ACTION: GLOBAL KILL SWITCH DEACTIVATED. System restored.")
+    return {"status": "success", "kill_switch_active": payload.active}
+
+@router.post("/system/flush-cache")
+async def flush_cache():
+    """Nuke all Redis keys (cache & telemetry). Extreme warning."""
+    if redis_client.is_connected and redis_client.redis:
+        try:
+            await redis_client.redis.flushdb()
+            logger.critical("Admin action: Redis FLUSHDB executed.")
+            return {"status": "success", "msg": "Redis flushed"}
+        except Exception as e:
+            raise HTTPException(status_code=500, detail=str(e))
+    raise HTTPException(status_code=503, detail="Redis offline")
+
+@router.get("/config-state")
+async def get_config_state():
+    """Retrieve current dynamic config state for Dashboard UI sync."""
+    return {
+        "kill_switch_active": router_module.KILL_SWITCH_ACTIVE,
+        "banned_models": list(router_module.banned_models),
+    }

app/api/metrics.py

@@ -1,6 +1,7 @@
 from fastapi import APIRouter
 from app.core.telemetry import telemetry
 from app.services.chat.api.llm_router import circuit_registry, llm_router
+import app.services.chat.api.llm_router as router_module
 from app.core.redis_client import redis_client
 from app.services.chat.memory.memory_manager import session_memory
 import time
@@ -52,6 +53,8 @@ async def get_metrics():
         "uptime_seconds": int(time.time() - START_TIME),
         "active_sessions": session_memory.get_stats().get("active_sessions", 0),
         "python_version": sys.version.split(" ")[0],
+        "kill_switch_active": router_module.KILL_SWITCH_ACTIVE,
+        "banned_models": list(router_module.banned_models),
     }
 
     return {

app/services/chat/api/llm_router.py

@@ -32,6 +32,16 @@ from contextlib import asynccontextmanager
 
 logger = logging.getLogger(__name__)
 
+# ──────────────────────────────────────────────────────────────────────────────
+# Admin Control Primitives — controllable at runtime via /api/admin/*
+# ──────────────────────────────────────────────────────────────────────────────
+
+# Global kill switch — when True, ALL LLM calls are rejected immediately with 503
+KILL_SWITCH_ACTIVE: bool = False
+
+# Banned model signatures — any model in this set is skipped by all providers
+banned_models: set[str] = set()
+
 
 def _get_model_name(client) -> str:
     """Extract the first active model name from a provider client.
@@ -350,6 +360,26 @@ class LLMRouter:
         # Local fallback
         self._local_active_requests = max(0, self._local_active_requests - 1)
 
+    def set_bulkhead_limit(self, provider: str, new_limit: int) -> bool:
+        """Dynamically resize the concurrency semaphore for a provider (Admin API).
+
+        Creates or replaces the semaphore with a fresh asyncio.Semaphore.
+        Returns True if applied, False if the provider is unknown.
+        """
+        known = [n for n, _ in self.providers]
+        if provider not in known:
+            return False
+        new_limit = max(1, new_limit)  # Always allow at least 1 slot
+        self._concurrency_budget[provider] = new_limit
+        # Replace the semaphore — existing in-flight calls on the old semaphore
+        # will complete normally; new acquisitions use the fresh one.
+        self._semaphores[provider] = asyncio.Semaphore(new_limit)
+        set_bulkhead_capacity(provider, new_limit)
+        logger.warning(
+            f"[Admin] Bulkhead for '{provider}' resized to {new_limit} concurrent slots."
+        )
+        return True
+
     def get_bulkhead_metrics(self) -> dict:
         data = {}
         for name, sem in self._semaphores.items():
@@ -443,6 +473,11 @@ class LLMRouter:
         access_token: Optional[str] = None,
     ) -> Tuple[str, List[ChatMessage], Optional[ToolConfirmationRequest]]:
         
+        global KILL_SWITCH_ACTIVE, banned_models
+        if KILL_SWITCH_ACTIVE:
+            logger.critical("🚨 Global Kill Switch is ACTIVE. Rejecting llm call_api.")
+            raise HTTPException(status_code=503, detail="Service administrative pause.")
+
         request_id = str(uuid.uuid4())[:8]
         last_error = None
         token, used_redis = await self._incr_requests()
@@ -462,6 +497,11 @@ class LLMRouter:
                 call_api_bulkhead_skipped: list[str] = []
 
                 for name, client in call_api_scored:
+                    model_signature = _get_model_name(client)
+                    if model_signature in banned_models:
+                        logger.warning(f"Skipping {name} — model {model_signature} is globally BANNED.")
+                        continue
+
                     cb = circuit_registry.get(name, failure_threshold=3, recovery_timeout=30)
                     if not cb.can_execute():
                         logger.debug(f"Circuit [{name}] is OPEN — skipping")

app/static/dashboard.html

@@ -10,16 +10,18 @@
     <script src="https://cdn.jsdelivr.net/npm/chart.js@4.4.7/dist/chart.umd.min.js"></script>
     <style>
         :root {
-            --bg-base: #0f111a;
-            --bg-panel: #171a23;
-            --bg-hover: #222633;
-            --border: #2b3040;
+            --bg-base: #0b0d14;
+            --bg-panel: #131620;
+            --bg-hover: #1c202f;
+            --border: #242938;
             --text-main: #f8fafc;
             --text-dim: #94a3b8;
-            --text-muted: #64748b;
+            --text-muted: #475569;
             --accent: #3b82f6;
+            --accent-hover: #2563eb;
             --critical: #ef4444;
             --warning: #f59e0b;
+            --warning-hover: #d97706;
             --ok: #10b981;
             --font-ui: 'Inter', sans-serif;
             --font-mono: 'Fira Code', monospace;
@@ -27,274 +29,253 @@
 
         * { margin: 0; padding: 0; box-sizing: border-box; }
 
-        body {
-            font-family: var(--font-ui);
-            background: var(--bg-base);
-            color: var(--text-main);
-            display: flex;
-            height: 100vh;
-            overflow: hidden;
-        }
-
-        /* ─── Sidebar ─── */
-        .sidebar {
-            width: 280px;
-            background: var(--bg-panel);
-            border-right: 1px solid var(--border);
-            display: flex;
-            flex-direction: column;
-            z-index: 10;
-        }
-
-        .brand {
-            padding: 24px;
-            border-bottom: 1px solid var(--border);
-            display: flex;
-            align-items: center;
-            gap: 12px;
-        }
-
-        .brand-icon {
-            width: 36px; height: 36px;
-            background: linear-gradient(135deg, var(--accent), #8b5cf6);
-            border-radius: 8px;
-            display: flex; align-items: center; justify-content: center;
-            font-weight: 800; font-size: 18px;
-            box-shadow: 0 4px 12px rgba(59, 130, 246, 0.4);
-        }
-
-        .sys-info {
-            padding: 24px;
-            flex: 1;
-        }
-
-        .sys-item {
-            margin-bottom: 20px;
-        }
-
-        .sys-label {
-            font-size: 11px;
-            text-transform: uppercase;
-            letter-spacing: 1px;
-            color: var(--text-muted);
-            margin-bottom: 6px;
-        }
-
-        .sys-val {
-            font-size: 14px;
-            font-weight: 600;
-            font-family: var(--font-mono);
-        }
-
-        /* ─── Main Content ─── */
-        .main {
-            flex: 1;
-            display: flex;
-            flex-direction: column;
-            overflow-y: auto;
-            position: relative;
-        }
-
-        .top-bar {
-            padding: 16px 32px;
-            background: rgba(23, 26, 35, 0.8);
-            backdrop-filter: blur(10px);
-            border-bottom: 1px solid var(--border);
-            display: flex;
-            justify-content: space-between;
-            align-items: center;
-            position: sticky;
-            top: 0;
-            z-index: 20;
-        }
-
-        .badge-status {
-            padding: 6px 12px;
-            border-radius: 20px;
-            font-size: 12px; font-weight: 600;
-            display: flex; align-items: center; gap: 8px;
-        }
-
-        .badge-status.healthy { background: rgba(16, 185, 129, 0.1); color: var(--ok); border: 1px solid rgba(16, 185, 129, 0.3); }
-        .badge-status.degraded { background: rgba(245, 158, 11, 0.1); color: var(--warning); border: 1px solid rgba(245, 158, 11, 0.3); }
-
-        .dashboard-content {
-            padding: 32px;
-            max-width: 1600px;
-            margin: 0 auto;
-            width: 100%;
-        }
-
-        .section-title {
-            font-size: 14px; text-transform: uppercase; letter-spacing: 1px;
-            color: var(--text-dim); margin: 32px 0 16px; border-bottom: 1px solid var(--border);
-            padding-bottom: 8px; display: flex; justify-content: space-between;
-        }
-
-        /* ─── Grids ─── */
+        body { font-family: var(--font-ui); background: var(--bg-base); color: var(--text-main); display: flex; height: 100vh; overflow: hidden; }
+
+        /* ─── Layout ─── */
+        .sidebar { width: 300px; background: var(--bg-panel); border-right: 1px solid var(--border); display: flex; flex-direction: column; z-index: 10; box-shadow: 4px 0 24px rgba(0,0,0,0.2); }
+        .main { flex: 1; display: flex; flex-direction: column; overflow-y: auto; position: relative; }
+        
+        /* ─── Typography & Utilities ─── */
+        .h-title { font-weight: 700; font-size: 16px; letter-spacing: -0.3px; }
+        .section-title { font-size: 15px; font-weight: 600; text-transform: uppercase; letter-spacing: 0.5px; color: var(--text-dim); margin: 32px 0 16px; border-bottom: 1px solid var(--border); padding-bottom: 12px; display: flex; justify-content: space-between; align-items:flex-end; }
+        .mono { font-family: var(--font-mono); }
+        .flex { display: flex; } .items-center { align-items: center; } .gap-2 { gap: 8px; } .justify-between { justify-content: space-between; }
+        
+        /* ─── Brand ─── */
+        .brand { padding: 24px; border-bottom: 1px solid var(--border); display: flex; align-items: center; gap: 14px; }
+        .brand-icon { width: 40px; height: 40px; background: linear-gradient(135deg, var(--critical), #8b5cf6); border-radius: 8px; display: flex; align-items: center; justify-content: center; font-weight: 800; font-size: 18px; box-shadow: 0 4px 16px rgba(239, 68, 68, 0.4); text-shadow: 0 2px 4px rgba(0,0,0,0.3); }
+
+        /* ─── Sidebar Items ─── */
+        .sys-info { padding: 24px; flex: 1; overflow-y: auto; }
+        .sys-item { margin-bottom: 24px; background: rgba(0,0,0,0.2); padding: 12px; border-radius: 8px; border: 1px solid var(--border); }
+        .sys-label { font-size: 11px; text-transform: uppercase; letter-spacing: 1px; color: var(--text-muted); margin-bottom: 8px; font-weight: 600;}
+        .sys-val { font-size: 15px; font-weight: 600; font-family: var(--font-mono); color: var(--accent); }
+
+        /* ─── Buttons ─── */
+        .btn { background: var(--bg-hover); color: var(--text-main); border: 1px solid var(--border); padding: 8px 14px; border-radius: 6px; font-size: 12px; font-weight: 600; cursor: pointer; transition: all 0.2s cubic-bezier(0.4, 0, 0.2, 1); display: inline-flex; align-items: center; gap: 6px; letter-spacing: 0.3px; }
+        .btn:hover { background: #2a3040; border-color: #3f475e; transform: translateY(-1px); }
+        .btn:active { transform: translateY(1px); }
+        .btn-crit { background: rgba(239,68,68,0.1); border-color: rgba(239,68,68,0.3); color: var(--critical); }
+        .btn-crit:hover { background: rgba(239,68,68,0.2); border-color: var(--critical); }
+        .btn-ok { background: rgba(16,185,129,0.1); border-color: rgba(16,185,129,0.3); color: var(--ok); }
+        .btn-ok:hover { background: rgba(16,185,129,0.2); border-color: var(--ok); }
+        .btn-warn { background: rgba(245,158,11,0.1); border-color: rgba(245,158,11,0.3); color: var(--warning); }
+        .btn-warn:hover { background: rgba(245,158,11,0.2); border-color: var(--warning); }
+
+        /* ─── Dashboard ─── */
+        .top-bar { padding: 16px 32px; background: rgba(19, 22, 32, 0.85); backdrop-filter: blur(12px); border-bottom: 1px solid var(--border); display: flex; justify-content: space-between; align-items: center; position: sticky; top: 0; z-index: 20; }
+        .dashboard-content { padding: 32px; max-width: 1600px; margin: 0 auto; width: 100%; }
+        
+        /* ─── Grids & Panels ─── */
         .grid-4 { display: grid; grid-template-columns: repeat(4, 1fr); gap: 16px; }
         .grid-2 { display: grid; grid-template-columns: 1fr 1fr; gap: 16px; }
-        .grid-3 { display: grid; grid-template-columns: repeat(3, 1fr); gap: 16px; }
-
-        /* ─── Panel ─── */
-        .panel {
-            background: var(--bg-panel);
-            border: 1px solid var(--border);
-            border-radius: 8px;
-            padding: 20px;
-            transition: border-color 0.2s;
-        }
-        .panel:hover { border-color: #3f475e; }
+        .panel { background: var(--bg-panel); border: 1px solid var(--border); border-radius: 10px; padding: 24px; transition: border-color 0.2s, box-shadow 0.2s; box-shadow: 0 4px 6px -1px rgba(0, 0, 0, 0.1); }
+        .panel:hover { border-color: #3f475e; box-shadow: 0 10px 15px -3px rgba(0, 0, 0, 0.1); }
 
-        .kpi-val { font-size: 32px; font-weight: 700; margin: 8px 0 4px; font-family: var(--font-mono); }
-        .kpi-lbl { font-size: 12px; color: var(--text-dim); text-transform: uppercase; letter-spacing: 0.5px; }
-        .kpi-sub { font-size: 12px; color: var(--text-muted); }
+        .kpi-val { font-size: 36px; font-weight: 700; margin: 12px 0 6px; font-family: var(--font-mono); letter-spacing: -1px; }
+        .kpi-lbl { font-size: 13px; color: var(--text-dim); font-weight: 600; text-transform: uppercase; letter-spacing: 0.5px; }
+        .kpi-sub { font-size: 12px; color: var(--text-muted); font-weight: 500; }
 
         /* ─── Tables ─── */
-        table { width: 100%; border-collapse: collapse; font-size: 13px; }
-        th { text-align: left; padding: 12px; border-bottom: 1px solid var(--border); color: var(--text-dim); font-weight: 600; text-transform: uppercase; font-size: 11px; letter-spacing: 0.5px; }
-        td { padding: 12px; border-bottom: 1px solid var(--border); font-family: var(--font-mono); }
+        table { width: 100%; border-collapse: separate; border-spacing: 0; font-size: 13px; }
+        th { text-align: left; padding: 16px; border-bottom: 1px solid var(--border); color: var(--text-dim); font-weight: 600; text-transform: uppercase; font-size: 11px; letter-spacing: 0.5px; background: rgba(0,0,0,0.2); }
+        td { padding: 16px; border-bottom: 1px solid var(--border); vertical-align: middle; }
         tr:last-child td { border-bottom: none; }
         tr:hover td { background: var(--bg-hover); }
 
-        /* ─── Elements ─── */
-        .status-dot {
-            display: inline-block; width: 8px; height: 8px; border-radius: 50%; margin-right: 6px;
-        }
-        .bg-ok { background: var(--ok); box-shadow: 0 0 8px rgba(16, 185, 129, 0.4); }
-        .bg-warn { background: var(--warning); box-shadow: 0 0 8px rgba(245, 158, 11, 0.4); }
-        .bg-crit { background: var(--critical); box-shadow: 0 0 8px rgba(239, 68, 68, 0.4); }
+        /* ─── Utilities ─── */
+        .status-dot { display: inline-block; width: 8px; height: 8px; border-radius: 50%; margin-right: 6px; }
+        .bg-ok { background: var(--ok); box-shadow: 0 0 8px rgba(16, 185, 129, 0.6); }
+        .bg-warn { background: var(--warning); box-shadow: 0 0 8px rgba(245, 158, 11, 0.6); }
+        .bg-crit { background: var(--critical); box-shadow: 0 0 8px rgba(239, 68, 68, 0.6); }
         .text-ok { color: var(--ok); } .text-warn { color: var(--warning); } .text-crit { color: var(--critical); }
-
-        .progress { height: 4px; background: var(--border); border-radius: 2px; overflow: hidden; margin-top: 8px; }
-        .progress-bar { height: 100%; transition: width 0.3s; }
-
-        .chart-wrap { height: 240px; margin-top: 16px; }
-
-        .tag { padding: 4px 8px; border-radius: 4px; font-size: 11px; font-weight: 600; background: var(--bg-hover); border: 1px solid var(--border); }
+        
+        .progress { height: 6px; background: rgba(0,0,0,0.4); border-radius: 3px; overflow: hidden; margin-top: 8px; border: 1px solid var(--border);}
+        .progress-bar { height: 100%; transition: width 0.4s cubic-bezier(0.4, 0, 0.2, 1); }
+        .chart-wrap { height: 260px; margin-top: 16px; }
+        .tag { padding: 4px 10px; border-radius: 6px; font-size: 11px; font-weight: 600; background: var(--bg-base); border: 1px solid var(--border); font-family: var(--font-mono); }
+
+        /* ─── Modals & Overlays ─── */
+        .overlay { position: fixed; top: 0; left: 0; width: 100%; height: 100%; background: rgba(11, 13, 20, 0.85); backdrop-filter: blur(8px); display: none; align-items: center; justify-content: center; z-index: 9999; opacity: 0; transition: opacity 0.2s; }
+        .overlay.active { display: flex; opacity: 1; }
+        
+        .modal { background: var(--bg-panel); border: 1px solid var(--border); padding: 32px; border-radius: 12px; width: 420px; box-shadow: 0 25px 50px -12px rgba(0, 0, 0, 0.5); transform: translateY(20px); transition: transform 0.3s cubic-bezier(0.34, 1.56, 0.64, 1); }
+        .overlay.active .modal { transform: translateY(0); }
+        
+        .modal-title { font-size: 18px; font-weight: 700; margin-bottom: 8px; display: flex; align-items: center; gap: 8px; }
+        .modal-desc { font-size: 13px; color: var(--text-dim); margin-bottom: 24px; line-height: 1.5; }
+        
+        .form-group { margin-bottom: 16px; text-align: left; }
+        .form-label { display: block; font-size: 12px; font-weight: 600; color: var(--text-dim); margin-bottom: 6px; }
+        .form-input { width: 100%; padding: 12px; background: rgba(0,0,0,0.2); border: 1px solid var(--border); color: white; border-radius: 6px; font-family: var(--font-mono); font-size: 14px; outline: none; transition: border-color 0.2s; }
+        .form-input:focus { border-color: var(--accent); box-shadow: 0 0 0 2px rgba(59,130,246,0.2); }
+        
+        .modal-actions { display: flex; justify-content: flex-end; gap: 12px; margin-top: 32px; }
+
+        /* Toast system */
+        #toastContainer { position: fixed; bottom: 32px; right: 32px; z-index: 10000; display: flex; flex-direction: column; gap: 12px; }
+        .toast { background: var(--bg-panel); border: 1px solid var(--border); color: white; padding: 14px 20px; border-radius: 8px; font-size: 13px; font-weight: 500; box-shadow: 0 10px 25px rgba(0,0,0,0.5); animation: toastIn 0.3s cubic-bezier(0.34, 1.56, 0.64, 1); display: flex; align-items: center; gap: 12px; min-width: 300px; }
+        @keyframes toastIn { from { transform: translateX(120%); opacity: 0; } to { transform: translateX(0); opacity: 1; } }
+
+        /* Danger Banner */
+        .danger-banner { background: rgba(239, 68, 68, 0.1); border: 1px solid rgba(239, 68, 68, 0.4); padding: 16px 24px; border-radius: 10px; margin-bottom: 32px; display: none; align-items: center; gap: 16px; color: var(--critical); font-weight: 700; font-size: 15px; animation: pulse-danger 2s infinite; letter-spacing: 0.5px; }
+        @keyframes pulse-danger { 0%,100% { box-shadow: 0 0 0px 0px rgba(239, 68, 68, 0.1); } 50% { box-shadow: 0 0 20px 8px rgba(239, 68, 68, 0.15); } }
+
+        /* Admin Controls specific */
+        .controls-wrapper { display: flex; flex-direction: column; gap: 8px; }
+        .controls-row { display: flex; gap: 8px; flex-wrap: wrap; }
     </style>
 </head>
 <body>
 
-    <!-- Sidebar -->
+    <!-- Auth Gate -->
+    <div id="authOverlay" class="overlay active" style="background:#0b0d14;">
+        <div class="modal" style="text-align: center;">
+            <div class="brand-icon" style="margin: 0 auto 20px;">🔐</div>
+            <h2 class="modal-title" style="justify-content: center;">Restricted Access</h2>
+            <p class="modal-desc">Enter your X-API-Key to unlock the Command Center.</p>
+            <input type="password" id="apiKeyIn" class="form-input" style="text-align:center; margin-bottom:24px;" placeholder="••••••••••••••••" onkeypress="if(event.key==='Enter') saveKey()">
+            <button class="btn btn-ok" style="width:100%; justify-content:center; padding:12px; font-size:14px;" onclick="saveKey()">Unlock Console</button>
+        </div>
+    </div>
+
+    <!-- Generic Modal for Tuning -->
+    <div id="tuneModal" class="overlay">
+        <div class="modal">
+            <h2 class="modal-title" id="tuneTitle">Tune Parameter</h2>
+            <p class="modal-desc" id="tuneDesc">Modify runtime constraints.</p>
+            <div id="tuneFields"></div>
+            <div class="modal-actions">
+                <button class="btn" onclick="closeModal()">Cancel</button>
+                <button class="btn btn-ok" id="tuneConfirm">Apply Changes</button>
+            </div>
+        </div>
+    </div>
+
+    <div id="toastContainer"></div>
+
     <aside class="sidebar">
         <div class="brand">
             <div class="brand-icon">A</div>
             <div>
-                <div style="font-weight: 700; font-size: 15px;">Awn Command</div>
-                <div style="font-size: 12px; color: var(--text-muted);">Admin NOC View</div>
+                <div style="font-weight: 800; font-size: 16px; letter-spacing: -0.5px;">Awn Command</div>
+                <div style="font-size: 12px; color: var(--accent); font-weight: 600; letter-spacing: 0.5px;">NOC CORE v2.0</div>
             </div>
         </div>
         <div class="sys-info">
             <div class="sys-item">
-                <div class="sys-label">Uptime</div>
+                <div class="sys-label">System Uptime</div>
                 <div class="sys-val" id="sysUptime">—</div>
             </div>
             <div class="sys-item">
-                <div class="sys-label">Python Env</div>
+                <div class="sys-label">Runtime Env</div>
                 <div class="sys-val" id="sysPython">—</div>
             </div>
             <div class="sys-item">
                 <div class="sys-label">Active Chat Sessions</div>
-                <div class="sys-val" id="sysSessions">—</div>
+                <div class="sys-val" id="sysSessions" style="color:var(--text-main)">—</div>
             </div>
-            <div class="sys-item">
-                <div class="sys-label">Redis State</div>
-                <div class="sys-val" id="sysRedis">—</div>
+            
+            <div style="margin-top: 48px; border-top: 1px dashed var(--border); padding-top: 32px;">
+                <div class="sys-label" style="color:var(--critical); display:flex; align-items:center; gap:6px;">⚠️ Danger Zone</div>
+                <div style="display:flex; flex-direction:column; gap:16px; margin-top:20px;">
+                    <button class="btn btn-crit" style="padding:12px; justify-content:center;" onclick="adminCmd('POST', '/api/ai/admin/system/killswitch', {active:!globalKillSwitch})">
+                        <span id="killBtnLbl">⚡ Trigger Kill Switch</span>
+                    </button>
+                    <button class="btn" style="padding:12px; justify-content:center; color:var(--text-dim);" onclick="if(confirm('Warning: This destroys all telemetry histories. Proceed?')) adminCmd('POST', '/api/ai/admin/system/flush-cache')">
+                        🗑 Nuke Redis Cache
+                    </button>
+                    <button class="btn" onclick="logout()" style="justify-content:center; margin-top:32px; background:transparent; border-color:transparent; color:var(--text-dim);">🔒 Lock Console</button>
+                </div>
             </div>
         </div>
     </aside>
 
-    <!-- Main -->
     <main class="main">
         <header class="top-bar">
-            <div class="badge-status healthy" id="mainStatus">
-                <div class="status-dot bg-ok"></div>
-                <span id="mainStatusText">System Healthy</span>
-            </div>
-            <div style="font-size:12px; color:var(--text-dim);">
-                Last Sync: <span id="syncTime">—</span>
+            <div class="h-title">Global LLM Routing Engine</div>
+            <div style="font-size:12px; color:var(--text-dim); display:flex; gap:20px; align-items:center; font-weight:500;">
+                <span style="display:flex; align-items:center; gap:6px;"><div class="status-dot bg-ok"></div> Secure</span>
+                <span style="background:rgba(0,0,0,0.3); padding:4px 10px; border-radius:12px; border:1px solid var(--border);">Last Sync: <span id="syncTime" class="mono">Never</span></span>
             </div>
         </header>
 
         <div class="dashboard-content">
             
-            <!-- KPIs -->
-            <div class="grid-4" style="margin-bottom: 32px;">
+            <div class="danger-banner" id="killSwitchBanner">
+                <div style="font-size:24px;">🚫</div>
+                <div>
+                    <div style="font-size:18px; margin-bottom:4px;">SYSTEM HALTED</div>
+                    <div style="font-size:13px; font-weight:500; opacity:0.9;">Global Kill Switch is active. All inbound LLM routing is returning 503 artificially.</div>
+                </div>
+            </div>
+
+            <div class="grid-4" style="margin-bottom: 40px;">
                 <div class="panel">
                     <div class="kpi-lbl">Router Score (Avg)</div>
-                    <div class="kpi-val" id="kpiScore">—</div>
-                    <div class="kpi-sub">Out of 100 max</div>
+                    <div class="kpi-val text-ok" id="kpiScore">—</div>
+                    <div class="kpi-sub">Out of 100 maximum</div>
                 </div>
                 <div class="panel">
                     <div class="kpi-lbl">Global Success Rate</div>
                     <div class="kpi-val" id="kpiSuccess">—</div>
-                    <div class="kpi-sub">Sliding window</div>
+                    <div class="kpi-sub">Sliding memory window</div>
                 </div>
                 <div class="panel">
-                    <div class="kpi-lbl">Provider Availability</div>
+                    <div class="kpi-lbl">Infrastructure Health</div>
                     <div class="kpi-val" id="kpiProviders">—</div>
-                    <div class="kpi-sub">Active endpoints</div>
+                    <div class="kpi-sub">Active nodes</div>
                 </div>
                 <div class="panel">
-                    <div class="kpi-lbl">Redis RTT (p50)</div>
-                    <div class="kpi-val" id="kpiRedisRtt">—</div>
-                    <div class="kpi-sub">Database latency</div>
+                    <div class="kpi-lbl">Redis Cache RTT</div>
+                    <div class="kpi-val mono text-accent" style="color:var(--accent)" id="kpiRedisRtt">—</div>
+                    <div class="kpi-sub" id="kpiRedisSub">Disconnected</div>
                 </div>
             </div>
 
-            <!-- Detailed Models Table -->
+            <!-- Provider Table with Full Actions -->
+            <div class="section-title">
+                <span>Provider Matrix & Routing Constraints</span>
+            </div>
+            <div class="panel" style="padding:0; overflow-x:auto;">
+                <table id="providersTable">
+                    <thead>
+                        <tr>
+                            <th>Integration Node</th>
+                            <th>Status & Breaker</th>
+                            <th>Score / SR</th>
+                            <th>Concurrency (Bulkhead)</th>
+                            <th style="min-width: 320px;">Administrative Restraints</th>
+                        </tr>
+                    </thead>
+                    <tbody><!-- JS --></tbody>
+                </table>
+            </div>
+
+            <!-- Model Banlist Table -->
             <div class="section-title">
-                Detailed Model Telemetry
-                <span style="font-size:11px; font-weight:normal;">Aggregated runtime empirical metrics</span>
+                <span>Micro-Model Precision Control</span>
             </div>
             <div class="panel" style="padding:0; overflow-x:auto;">
                 <table id="modelsTable">
                     <thead>
                         <tr>
                             <th>Model Signature</th>
+                            <th>Traffic Volume</th>
                             <th>Success %</th>
-                            <th>Avg Latency (ms)</th>
-                            <th>Avg TTFT (ms)</th>
-                            <th>Calls ✓</th>
-                            <th>Errors ✗</th>
-                            <th>Last Used</th>
+                            <th>P50 Lat/TTFT</th>
+                            <th>Routing Rule</th>
                         </tr>
                     </thead>
-                    <tbody>
-                        <!-- Populated by JS -->
-                    </tbody>
+                    <tbody><!-- JS --></tbody>
                 </table>
             </div>
 
-            <!-- Provider Health & Breakers -->
-            <div class="section-title">Routing Protocol Status & Protection</div>
-            <div class="grid-2">
-                <div class="panel">
-                    <div class="kpi-lbl" style="margin-bottom:16px;">Provider Scoring Matrix</div>
-                    <div id="providerRanks">
-                        <!-- JS -->
-                    </div>
-                </div>
-                <div class="panel">
-                    <div class="kpi-lbl" style="margin-bottom:16px;">Circuit Breakers & Bulkhead Limits</div>
-                    <div id="protectionRanks">
-                        <!-- JS -->
-                    </div>
-                </div>
-            </div>
-
             <!-- Charts -->
-            <div class="section-title">Live Trend Analysis</div>
+            <div class="section-title" style="margin-top:40px;">Real-time Telemetry Profiling</div>
             <div class="grid-2">
-                <div class="panel">
-                    <div class="kpi-lbl">Latency Profiling (ms)</div>
-                    <div class="chart-wrap"><canvas id="chartLat"></canvas></div>
-                </div>
-                <div class="panel">
-                    <div class="kpi-lbl">Success Rate Discrepancy (%)</div>
-                    <div class="chart-wrap"><canvas id="chartSr"></canvas></div>
-                </div>
+                <div class="panel"><div class="kpi-lbl">Latency Dist. (ms)</div><div class="chart-wrap"><canvas id="chartLat"></canvas></div></div>
+                <div class="panel"><div class="kpi-lbl">Success Rate Discrepancy (%)</div><div class="chart-wrap"><canvas id="chartSr"></canvas></div></div>
             </div>
 
         </div>
@@ -302,210 +283,240 @@
 
     <script>
         const METRICS_URL = '/api/ai/metrics';
+        let API_KEY = localStorage.getItem('awn_api_key');
         let chartLat, chartSr;
         const trendData = {};
+        let globalKillSwitch = false;
+        let globalBannedModels = [];
+
+        // Auth
+        if(API_KEY) { document.getElementById('authOverlay').classList.remove('active'); setTimeout(() => document.getElementById('authOverlay').style.display='none', 200); }
+        function saveKey() {
+            const v = document.getElementById('apiKeyIn').value.trim();
+            if(v) { API_KEY = v; localStorage.setItem('awn_api_key', v); document.getElementById('authOverlay').classList.remove('active'); setTimeout(() => document.getElementById('authOverlay').style.display='none', 200); fetchLoop(); }
+        }
+        function logout() {
+            localStorage.removeItem('awn_api_key'); API_KEY = null; 
+            document.getElementById('authOverlay').style.display='flex';
+            setTimeout(() => document.getElementById('authOverlay').classList.add('active'), 10);
+        }
 
-        function formatTime(seconds) {
-            if (seconds < 60) return `${Math.floor(seconds)}s`;
-            if (seconds < 3600) return `${Math.floor(seconds/60)}m`;
-            return `${(seconds/3600).toFixed(1)}h`;
+        // Toasts
+        function showToast(msg, type='ok') {
+            const c = document.getElementById('toastContainer');
+            const el = document.createElement('div'); el.className = 'toast';
+            el.innerHTML = `<div class="status-dot ${type==='ok'?'bg-ok':'bg-crit'}" style="box-shadow:none;transform:scale(1.2)"></div> ${msg}`;
+            c.appendChild(el);
+            setTimeout(() => { el.style.opacity = '0'; el.style.transform = 'translateX(100%)'; setTimeout(()=>el.remove(), 300); }, 4000);
         }
 
-        function initCharts() {
-            const cfg = {
-                responsive: true, maintainAspectRatio: false,
-                animation: false,
-                plugins: { legend: { labels: { color: '#94a3b8', font:{family:'Inter'} } }, tooltip: { mode: 'index' } },
-                scales: { 
-                    x: { display: false }, 
-                    y: { grid: { color: '#2b3040' }, border:{display:false}, ticks: { color: '#64748b' } } 
-                }
+        // Execution
+        async function adminCmd(method, path, payload=null) {
+            try {
+                const res = await fetch(path, { method: method, headers: { 'X-API-Key': API_KEY, 'Content-Type': 'application/json' }, body: payload ? JSON.stringify(payload) : null });
+                const data = await res.json();
+                if(res.ok) { showToast(`Command executed successfully`, 'ok'); fetchLoop(); closeModal(); } 
+                else { showToast(`Failed: ${data.detail || res.statusText}`, 'crit'); if(res.status === 401) logout(); }
+            } catch(e) { showToast(`Network Hook Error: ${e.message}`, 'crit'); }
+        }
+
+        // Modal Logic
+        const modal = document.getElementById('tuneModal');
+        let currentModalAction = null;
+        
+        function closeModal() { modal.classList.remove('active'); setTimeout(()=>modal.style.display='none', 200); }
+        function openTuneBulkhead(name, currentLimit) {
+            document.getElementById('tuneTitle').innerHTML = `⚙️ Tune Bulkhead: <span class="tag">${name}</span>`;
+            document.getElementById('tuneDesc').textContent = 'Adjust the concurrency semaphore limit instantly.';
+            document.getElementById('tuneFields').innerHTML = `<div class="form-group"><label class="form-label">Concurrency Limit</label><input type="number" id="inpBulk" class="form-input" value="${currentLimit}" min="1"></div>`;
+            modal.style.display = 'flex'; setTimeout(()=>modal.classList.add('active'), 10);
+            
+            document.getElementById('tuneConfirm').onclick = () => {
+                const v = parseInt(document.getElementById('inpBulk').value);
+                if(v>0) adminCmd('POST', `/api/ai/admin/provider/${name}/bulkhead`, {limit:v});
+            };
+        }
+        function openTuneCircuit(name, currentFail, currentTimeout) {
+            document.getElementById('tuneTitle').innerHTML = `🛡️ Tune Circuit: <span class="tag">${name}</span>`;
+            document.getElementById('tuneDesc').textContent = 'Modify dynamic fault tolerance thresholds.';
+            document.getElementById('tuneFields').innerHTML = `
+                <div class="form-group"><label class="form-label">Failure Threshold (Strikes)</label><input type="number" id="inpFail" class="form-input" value="${currentFail}" min="1"></div>
+                <div class="form-group"><label class="form-label">Base Recovery Timeout (Seconds)</label><input type="number" id="inpTimeout" class="form-input" value="${currentTimeout}" min="1"></div>
+            `;
+            modal.style.display = 'flex'; setTimeout(()=>modal.classList.add('active'), 10);
+            
+            document.getElementById('tuneConfirm').onclick = () => {
+                const tf = parseInt(document.getElementById('inpFail').value);
+                const rt = parseInt(document.getElementById('inpTimeout').value);
+                if(tf>0 && rt>0) adminCmd('POST', `/api/ai/admin/circuit/${name}/tune`, {failure_threshold: tf, recovery_timeout: rt});
             };
+        }
 
-            chartLat = new Chart(document.getElementById('chartLat'), {
-                type: 'line', data: { labels: [], datasets: [] }, options: cfg
-            });
 
-            const cfgSr = JSON.parse(JSON.stringify(cfg));
-            cfgSr.scales.y.min = 0; cfgSr.scales.y.max = 100;
-            chartSr = new Chart(document.getElementById('chartSr'), {
-                type: 'line', data: { labels: [], datasets: [] }, options: cfgSr
-            });
+        // Formatting
+        function formatTime(s) {
+            if (s < 60) return `${Math.floor(s)}s`;
+            if (s < 3600) return `${Math.floor(s/60)}m`;
+            if (s < 86400) return `${(s/3600).toFixed(1)}h`;
+            return `${(s/86400).toFixed(1)}d`;
+        }
+
+        // Charts
+        const cols = ['#3b82f6', '#8b5cf6', '#10b981', '#f59e0b', '#ec4899'];
+        function initCharts() {
+            const cfg = { responsive: true, maintainAspectRatio: false, animation: false, plugins: { legend: { labels: { color: '#94a3b8', font:{family:'Inter'} } }, tooltip: { mode: 'index', backgroundColor:'rgba(19,22,32,0.9)', titleColor:'#fff', bodyFont:{family:'Fira Code'}, borderColor:'#242938', borderWidth:1 } }, scales: { x: { display: false }, y: { grid: { color: '#242938' }, border:{display:false}, ticks: { color: '#475569', font:{family:'Fira Code'} } } } };
+            chartLat = new Chart(document.getElementById('chartLat'), { type: 'line', data: { labels: [], datasets: [] }, options: cfg });
+            const cfgSr = JSON.parse(JSON.stringify(cfg)); cfgSr.scales.y.min = 0; cfgSr.scales.y.max = 100;
+            chartSr = new Chart(document.getElementById('chartSr'), { type: 'line', data: { labels: [], datasets: [] }, options: cfgSr });
+        }
+        function drawCharts() {
+            const names = Object.keys(trendData); if(!names.length) return;
+            const lbl = new Date().toLocaleTimeString();
+            const maxL = Math.max(...names.map(n=>trendData[n].lat.length));
+            const x = Array(maxL).fill(''); x[x.length-1]=lbl;
+            chartLat.data.labels = x; chartSr.data.labels = x;
+            chartLat.data.datasets = names.map((n,i)=>({ label:n, data:trendData[n].lat, borderColor:cols[i%cols.length], backgroundColor:cols[i%cols.length]+'22', fill:true, borderWidth:2, pointRadius:0, tension:0.4 }));
+            chartSr.data.datasets = names.map((n,i)=>({ label:n, data:trendData[n].sr, borderColor:cols[i%cols.length], backgroundColor:cols[i%cols.length]+'22', fill:true, borderWidth:2, pointRadius:0, tension:0.4 }));
+            chartLat.update(); chartSr.update();
         }
 
-        function updateSys(system, redis) {
-            if(system) {
-                document.getElementById('sysUptime').textContent = formatTime(system.uptime_seconds);
-                document.getElementById('sysPython').textContent = `v${system.python_version}`;
-                document.getElementById('sysSessions').textContent = system.active_sessions;
+        // Render Functions
+        function renderSys(sys, redis) {
+            if(sys) {
+                document.getElementById('sysUptime').textContent = formatTime(sys.uptime_seconds);
+                document.getElementById('sysPython').textContent = `v${sys.python_version}`;
+                document.getElementById('sysSessions').textContent = sys.active_sessions;
+                
+                globalKillSwitch = sys.kill_switch_active;
+                globalBannedModels = sys.banned_models || [];
+                
+                document.getElementById('killSwitchBanner').style.display = globalKillSwitch ? 'flex' : 'none';
+                document.getElementById('killBtnLbl').textContent = globalKillSwitch ? '🟢 Heal System (Deactivate Kill)' : '⚡ Trigger Kill Switch';
             }
             if(redis) {
-                document.getElementById('sysRedis').innerHTML = redis.connected ? 
-                    `<span class="text-ok">Connected</span>` : `<span class="text-crit">Offline</span>`;
-                document.getElementById('kpiRedisRtt').textContent = redis.last_ping_ms ? `${redis.last_ping_ms}ms` : '—';
+                document.getElementById('kpiRedisSub').textContent = redis.connected ? 'OK / Connected' : 'Down';
+                document.getElementById('kpiRedisSub').className = redis.connected ? 'kpi-sub text-ok' : 'kpi-sub text-crit';
+                document.getElementById('kpiRedisRtt').textContent = redis.connected && redis.last_ping_ms ? `${redis.last_ping_ms}ms` : '—';
             }
         }
 
-        function updateModels(models) {
-            const tbody = document.querySelector('#modelsTable tbody');
-            tbody.innerHTML = '';
-            
-            // Sort by total calls descending
-            const entries = Object.entries(models || {}).sort((a,b) => b[1].total_calls - a[1].total_calls);
-            
-            entries.forEach(([name, m]) => {
-                const srClass = m.success_rate >= 95 ? 'text-ok' : (m.success_rate >= 80 ? 'text-warn' : 'text-crit');
-                const lastUnix = m.last_used_unix;
-                const lastUsedStr = lastUnix ? new Date(lastUnix * 1000).toLocaleTimeString() : 'Never';
-                
-                tbody.innerHTML += `
-                    <tr>
-                        <td><span class="tag">${name}</span></td>
-                        <td class="${srClass}">${m.success_rate.toFixed(1)}%</td>
-                        <td>${m.avg_latency_ms}</td>
-                        <td>${m.avg_ttft_ms || '—'}</td>
-                        <td class="text-ok">${m.total_calls - m.total_errors}</td>
-                        <td class="text-crit">${m.total_errors}</td>
-                        <td style="color:var(--text-muted)">${lastUsedStr}</td>
-                    </tr>
-                `;
-            });
-            if(entries.length === 0) tbody.innerHTML = `<tr><td colspan="7" style="text-align:center;">No telemetry data yet</td></tr>`;
-        }
+        function renderProviders(providers, breakers) {
+            const tbody = document.querySelector('#providersTable tbody');
+            tbody.innerHTML = ''; let tScore=0, tSr=0, act=0, cnt=0;
 
-        function updateProviders(providers, breakers) {
-            let totalScore = 0, totalSr = 0, count = 0, active = 0;
-            const pGrid = document.getElementById('providerRanks');
-            const protGrid = document.getElementById('protectionRanks');
-            pGrid.innerHTML = ''; protGrid.innerHTML = '';
-            
-            const timeLabel = new Date().toLocaleTimeString();
-
-            for (const [name, p] of Object.entries(providers)) {
-                count++;
-                totalScore += p.score;
-                // FIX: p.success_rate_window is already a percentage (0-100) from the python backend.
-                const sr = p.success_rate_window || 0; 
-                totalSr += sr;
-                
+            for(const [name, p] of Object.entries(providers)) {
+                cnt++; tScore+=p.score; tSr+=(p.success_rate_window||0);
                 const cb = breakers[name] || {};
-                if (cb.state === 'closed') active++;
-
-                // Build Trend Data
-                if(!trendData[name]) trendData[name] = { lat: [], sr: [] };
-                trendData[name].lat.push(p.latency_p50_ms || 0);
-                trendData[name].sr.push(sr);
-                if(trendData[name].lat.length > 30) { trendData[name].lat.shift(); trendData[name].sr.shift(); }
-
-                // UI: Provider Routing Rank
-                const scClass = p.score >= 120 ? 'text-ok' : (p.score >= 80 ? 'text-warn' : 'text-crit');
-                pGrid.innerHTML += `
-                    <div style="margin-bottom:16px;">
-                        <div style="display:flex; justify-content:space-between; margin-bottom:4px;">
-                            <span style="font-weight:600">${name}</span>
-                            <span class="${scClass}" style="font-family:var(--font-mono)">Score: ${p.score.toFixed(1)}</span>
-                        </div>
-                        <div style="display:flex; justify-content:space-between; font-size:11px; color:var(--text-dim);">
-                            <span>SR: ${sr.toFixed(1)}% | Latency: ${p.latency_p50_ms}ms</span>
-                            <span>Streak: ${p.consecutive_failures} fails</span>
-                        </div>
-                        <div class="progress"><div class="progress-bar ${scClass.replace('text','bg')}" style="width:${Math.min((p.score/200)*100, 100)}%"></div></div>
-                    </div>
-                `;
+                const disabled = p.circuit_state === 'disabled' || (cb.state === 'open' && p.score < 0);
+                if(cb.state === 'closed' && !disabled) act++;
 
-                // UI: Resilience (Breakers + Bulkhead)
-                const cbClass = cb.state === 'closed' ? 'bg-ok' : (cb.state === 'half_open' ? 'bg-warn' : 'bg-crit');
-                const bh = p.bulkhead || {};
-                const bhUsed = bh.configured ? (bh.configured - bh.available) : 0;
+                const stateUi = disabled ? `<span class="status-dot bg-crit"></span> <span class="text-crit" style="font-weight:600">DISABLED</span>` : 
+                    (cb.state==='closed' ? `<span class="status-dot bg-ok"></span> <span class="text-ok">CLOSED (Live)</span>` : `<span class="status-dot bg-warn"></span> <span class="text-warn" style="font-weight:600">OPEN/TRIP (${cb.seconds_until_retry||0}s)</span>`);
+
+                const bh = p.bulkhead||{};
+                const used = bh.configured ? bh.configured - bh.available : 0;
+                const perc = bh.configured ? Math.min((used/bh.configured)*100, 100) : 0;
                 
-                protGrid.innerHTML += `
-                    <div style="margin-bottom:16px; padding:12px; background:var(--bg-hover); border-radius:6px; border:1px solid var(--border);">
-                        <div style="display:flex; justify-content:space-between; margin-bottom:8px;">
-                            <div style="font-weight:600">${name}</div>
-                            <div class="tag" style="display:flex; align-items:center;">
-                                <div class="status-dot ${cbClass}"></div> ${cb.state || 'closed'}
+                tbody.innerHTML += `
+                    <tr style="background: ${disabled ? 'rgba(0,0,0,0.4)': 'transparent'}; opacity: ${disabled ? 0.7 : 1}">
+                        <td><div style="font-weight:700; font-size:14px; margin-bottom:4px;">${name}</div> <div class="tag">${cb.failure_threshold||3} thresh / ${cb.recovery_timeout_s||60}s t/o</div></td>
+                        <td>
+                            <div style="margin-bottom:6px;">${stateUi}</div> 
+                            <div style="font-size:11px; color:var(--text-muted); font-family:var(--font-mono);">Streak: ${p.consecutive_failures}x fails</div>
+                        </td>
+                        <td class="mono">
+                            <div style="margin-bottom:4px; font-weight:600" class="${p.score>=100?'text-ok':'text-warn'}">Sc: ${p.score.toFixed(1)}</div>
+                            <div style="font-size:12px;" class="${p.success_rate_window>=90?'text-ok':'text-crit'}">SR: ${(p.success_rate_window||0).toFixed(1)}%</div>
+                        </td>
+                        <td>
+                            <div class="flex justify-between" style="font-size:11px; margin-bottom:4px; font-weight:600; font-family:var(--font-mono);">
+                                <span>${used} Active</span> <span style="color:var(--text-dim)">Cap: ${bh.configured||'?'}</span>
                             </div>
-                        </div>
-                        <div style="display:flex; justify-content:space-between; font-size:12px; color:var(--text-dim); font-family:var(--font-mono);">
-                            <span>Bulkhead: ${bhUsed}/${bh.configured || '?'} slots</span>
-                            <span class="text-warning">${cb.state !== 'closed' ? `Retry in ${cb.seconds_until_retry}s` : 'Active'}</span>
-                        </div>
-                    </div>
+                            <div class="progress"><div class="progress-bar ${perc>80?'bg-crit':'bg-accent'}" style="width:${perc}%"></div></div>
+                        </td>
+                        <td>
+                            <div class="controls-wrapper">
+                                <div class="controls-row">
+                                    ${disabled 
+                                        ? `<button class="btn btn-ok" onclick="adminCmd('POST','/api/ai/admin/provider/${name}/enable')">✓ Enable</button>`
+                                        : `<button class="btn btn-crit" onclick="adminCmd('POST','/api/ai/admin/provider/${name}/disable')">⛔ Power Off</button>`}
+                                    <button class="btn btn-warn" onclick="adminCmd('POST','/api/ai/admin/circuit/${name}/trip')">🔪 Trip</button>
+                                    <button class="btn btn-ok" style="background:transparent;" onclick="adminCmd('POST','/api/ai/admin/circuit/${name}/reset')">↻ Reset</button>
+                                </div>
+                                <div class="controls-row">
+                                    <button class="btn" style="background:transparent; color:var(--text-dim)" onclick="openTuneCircuit('${name}', ${cb.failure_threshold||3}, ${cb.recovery_base_s||60})">⚙️ Tune Circuit</button>
+                                    <button class="btn" style="background:transparent; color:var(--text-dim)" onclick="openTuneBulkhead('${name}', ${bh.configured||16})">📊 Adjust Slots</button>
+                                </div>
+                            </div>
+                        </td>
+                    </tr>
                 `;
+                
+                // charts prep
+                if(!trendData[name]) trendData[name]={lat:[], sr:[]};
+                trendData[name].lat.push(p.latency_p50_ms||0); trendData[name].sr.push(p.success_rate_window||0);
+                if(trendData[name].lat.length>40) { trendData[name].lat.shift(); trendData[name].sr.shift(); }
             }
-
-            // Update main KPIs
-            document.getElementById('kpiProviders').textContent = `${active}/${count}`;
-            document.getElementById('kpiScore').textContent = count ? (totalScore/count).toFixed(0) : '—';
-            document.getElementById('kpiSuccess').textContent = count ? (totalSr/count).toFixed(1) + '%' : '—';
-
-            // Top banner status
-            const banner = document.getElementById('mainStatus');
-            const bTxt = document.getElementById('mainStatusText');
-            if(active === 0 && count > 0) {
-                banner.className = 'badge-status degraded';
-                banner.querySelector('.status-dot').className = 'status-dot bg-crit';
-                bTxt.textContent = 'ALL PROVIDERS DOWN - FATAL';
-                bTxt.style.color = 'var(--critical)';
-            } else if (active < count) {
-                banner.className = 'badge-status degraded';
-                banner.querySelector('.status-dot').className = 'status-dot bg-warn';
-                bTxt.textContent = `DEGRADED (${active}/${count} Online)`;
-                bTxt.style.color = 'var(--warning)';
-            } else {
-                banner.className = 'badge-status healthy';
-                banner.querySelector('.status-dot').className = 'status-dot bg-ok';
-                bTxt.textContent = 'System Healthy';
-            }
-
-            updateCharts(timeLabel);
+            
+            document.getElementById('kpiProviders').textContent = `${act}/${cnt}`;
+            document.getElementById('kpiScore').className = (cnt?(tScore/cnt):0) >= 100 ? 'kpi-val text-ok' : 'kpi-val text-warn';
+            document.getElementById('kpiScore').textContent = cnt ? (tScore/cnt).toFixed(0) : '—';
+            document.getElementById('kpiSuccess').textContent = cnt ? +(tSr/cnt).toFixed(1) + '%' : '—';
         }
 
-        function updateCharts(timeLabel) {
-            const names = Object.keys(trendData);
-            if(names.length === 0) return;
-            
-            const maxLen = Math.max(...names.map(n => trendData[n].lat.length));
-            const labels = Array(maxLen).fill('');
-            labels[labels.length-1] = timeLabel;
-
-            const colors = ['#3b82f6', '#8b5cf6', '#10b981', '#f59e0b'];
-
-            chartLat.data.labels = labels;
-            chartLat.data.datasets = names.map((n, i) => ({
-                label: n, data: trendData[n].lat, 
-                borderColor: colors[i%4], borderWidth: 2, pointRadius: 0, tension: 0.3
-            }));
-            chartLat.update();
-
-            chartSr.data.labels = labels;
-            chartSr.data.datasets = names.map((n, i) => ({
-                label: n, data: trendData[n].sr, 
-                borderColor: colors[i%4], borderWidth: 2, pointRadius: 0, tension: 0.3
-            }));
-            chartSr.update();
+        function renderModels(models) {
+            const tbody = document.querySelector('#modelsTable tbody');
+            tbody.innerHTML = '';
+            const entries = Object.entries(models||{}).sort((a,b)=>b[1].total_calls - a[1].total_calls);
+
+            entries.forEach(([name, m]) => {
+                const isBanned = globalBannedModels.includes(name);
+                const s = m.success_rate;
+                const srClass = s>=95 ? 'text-ok' : (s>=70 ? 'text-warn' : 'text-crit');
+                
+                tbody.innerHTML += `
+                    <tr style="background: ${isBanned ? 'rgba(0,0,0,0.3)': 'transparent'}; opacity: ${isBanned ? 0.6 : 1}">
+                        <td style="${isBanned ? 'text-decoration:line-through' : ''}">
+                            <div class="tag" style="background: ${isBanned ? 'var(--bg-hover)' : 'rgba(59,130,246,0.1)'}; color: ${isBanned ? 'var(--text-muted)' : 'var(--accent)'}; border-color: ${isBanned ? 'transparent' : 'rgba(59,130,246,0.2)'}">${name}</div>
+                        </td>
+                        <td class="mono">
+                            <span class="text-ok">✓ ${m.total_calls - m.total_errors}</span>
+                            <span style="color:var(--text-dim); margin:0 4px">|</span> 
+                            <span class="text-crit">✗ ${m.total_errors}</span>
+                        </td>
+                        <td class="mono ${srClass}" style="font-weight:600">${s.toFixed(1)}%</td>
+                        <td class="mono">
+                            <span style="color:var(--text-main)">${m.avg_latency_ms}ms</span> <span style="font-size:11px;color:var(--text-muted)">lat</span>
+                            <span style="color:var(--text-dim); margin:0 4px">/</span>
+                            <span style="color:var(--text-main)">${m.avg_ttft_ms||'—'}ms</span> <span style="font-size:11px;color:var(--text-muted)">ttft</span>
+                        </td>
+                        <td>
+                            ${isBanned 
+                                ? `<button class="btn btn-ok" onclick="adminCmd('POST','/api/ai/admin/model/unban',{model_name:'${name}'})">🔓 Remit Ban</button>`
+                                : `<button class="btn btn-crit" onclick="adminCmd('POST','/api/ai/admin/model/ban',{model_name:'${name}'})">⛔ Absolute Block</button>`}
+                        </td>
+                    </tr>
+                `;
+            });
         }
 
+        let loopTimer;
         async function fetchLoop() {
+            if(!API_KEY) return;
+            clearTimeout(loopTimer);
             try {
                 const res = await fetch(METRICS_URL);
                 const data = await res.json();
-                
                 document.getElementById('syncTime').textContent = new Date().toLocaleTimeString();
-                
-                updateSys(data.system, data.redis);
-                updateModels(data.models);
-                updateProviders(data.providers, data.breakers);
-            } catch (err) {
-                console.error("Dashboard sync failed", err);
-            }
-            setTimeout(fetchLoop, 5000);
+                renderSys(data.system, data.redis);
+                renderProviders(data.providers, data.breakers);
+                renderModels(data.models);
+                drawCharts();
+            } catch(e) { console.error("Sync Error", e); }
+            loopTimer = setTimeout(fetchLoop, 5000);
         }
 
-        document.addEventListener('DOMContentLoaded', () => {
-            initCharts();
-            fetchLoop();
-        });
+        document.addEventListener('DOMContentLoaded', () => { initCharts(); if(API_KEY) fetchLoop(); });
     </script>
 </body>
 </html>

main.py

@@ -123,8 +123,10 @@ def create_application() -> FastAPI:
 
     # Routes
     from app.api.api import public_router
+    from app.api.admin_ops import router as admin_router
     application.include_router(api_router, prefix="/api")
     application.include_router(public_router, prefix="/api")
+    application.include_router(admin_router, prefix="/api/ai")
     application.include_router(metrics_router)  # /metrics for Prometheus/Grafana
 
     @application.get("/", include_in_schema=False)