app.py

"""
app.py — ETP Distributed Scraper: Control Plane v5.1 (Item-Level Dashboard + Hash Status + Refresher Lock)
FastAPI + HTML/JS.

KEY CHANGES FROM v5:
  1. DASHBOARD: Uses fixed get_dashboard_item_stats() RPC
     - success_items and failed_items counted for ALL chunk statuses (not just completed/failed)
     - This ensures: done + pending = total ALWAYS HOLDS
     - No more 1,000-row .select() limit
     - Mathematically perfect item-level progress
  2. HASH STATUS: Shows current auth_hash status from system_tokens
     - When it was last refreshed
     - Whether it's active
  3. REFRESHER LOCK STATUS: Shows which hash_refresher instance is active
     - Instance ID
     - Last heartbeat time
     - If no refresher is running, shows warning
  4. Removed direct sub_tasks .select() — all stats via RPC
"""

import os, time, threading, logging
from datetime import datetime, timezone
from supabase import create_client, Client
from fastapi import FastAPI, Request
from fastapi.responses import HTMLResponse, JSONResponse
import uvicorn

logging.basicConfig(level=logging.INFO, format="%(asctime)s [%(levelname)s] %(message)s")
log = logging.getLogger("ControlPlane")

SUPABASE_URL       = os.environ["SUPABASE_URL"]
SUPABASE_KEY       = os.environ["SUPABASE_SERVICE_KEY"]
SURGEON_TIMEOUT    = int(os.environ.get("SURGEON_TIMEOUT_MINUTES", "5"))
WATCHDOG_INTERVAL  = 300

supabase: Client = create_client(SUPABASE_URL, SUPABASE_KEY)
app = FastAPI()

SERIES_MAP = {"0": "QL", "1": "ST", "2": "ML"}

# ─── BACKGROUND SURGEON (Auto-Cleaner) ──────────────────────────────────
def watchdog_loop():
    log.info("Watchdog started. Will permanently delete dead workers every 5 mins.")
    while True:
        try:
            res = supabase.rpc("surgeon_recover_dead_tasks", {"p_timeout_minutes": SURGEON_TIMEOUT}).execute()
            recovered = res.data or 0
            if recovered > 0:
                log.info(f"Surgeon recovered {recovered} stuck chunks and wiped dead workers.")
        except Exception as e:
            log.error(f"Surgeon error: {e}")
        time.sleep(WATCHDOG_INTERVAL)

threading.Thread(target=watchdog_loop, daemon=True).start()

# ─── API ROUTES ────────────────────────────────────────────────
@app.get("/api/dashboard")
def api_dashboard():
    # 1. Fetch tasks (few rows, no limit issue)
    tasks = supabase.table("main_tasks").select("*").order("priority").order("created_at").execute().data or []

    # 2. Fetch item-level stats via RPC (server-side aggregation, no row limit)
    #    Fixed RPC: success_items + failed_items counted for ALL statuses
    #    This ensures: done + pending = total ALWAYS
    stats_data = supabase.rpc("get_dashboard_item_stats").execute().data or []
    stats_map = {}
    for s in stats_data:
        stats_map[s["main_task_id"]] = s

    # 3. Compute global item-level stats
    global_stats = {"total_items": 0, "pending_items": 0, "in_progress_items": 0, "success_items": 0, "failed_items": 0}
    for t in tasks:
        s = stats_map.get(t["id"], {})
        t["item_stats"] = {
            "total_items": s.get("total_items", 0) or 0,
            "success_items": s.get("success_items", 0) or 0,
            "failed_items": s.get("failed_items", 0) or 0,
            "in_progress_items": s.get("in_progress_items", 0) or 0,
            "pending_items": s.get("pending_items", 0) or 0,
        }
        for key in global_stats:
            global_stats[key] += t["item_stats"].get(key, 0)

    # 4. Fetch and filter live workers (HIDE IF DEAD > 5 MINS)
    raw_workers = supabase.table("workers").select("*").order("started_at", desc=True).execute().data or []
    live_workers = []
    now_utc = datetime.now(timezone.utc)

    for w in raw_workers:
        try:
            hb_str = w["last_heartbeat"].replace("Z", "+00:00")
            hb_time = datetime.fromisoformat(hb_str)
            seconds_dead = (now_utc - hb_time).total_seconds()
            if seconds_dead <= 300:
                w["seconds_dead"] = int(seconds_dead)
                live_workers.append(w)
        except Exception:
            pass

    # 5. Fetch System State
    sys_config = supabase.table("system_config").select("*").eq("id", 1).execute().data
    global_paused = sys_config[0]["is_globally_paused"] if sys_config else False

    # 6. Fetch Auth Hash Status
    hash_info = None
    try:
        hash_res = supabase.rpc("get_active_auth_hash").execute()
        if hash_res.data and len(hash_res.data) > 0:
            hash_info = hash_res.data[0]
    except Exception:
        pass

    # 7. Fetch Refresher Lock Status
    refresher_info = None
    if sys_config:
        sc = sys_config[0]
        if sc.get("active_refresher_id"):
            refresher_info = {
                "instance_id": sc["active_refresher_id"],
                "last_heartbeat": sc.get("refresher_last_heartbeat"),
            }

    return JSONResponse({
        "tasks": tasks,
        "workers": live_workers,
        "global_stats": global_stats,
        "hash_info": hash_info,
        "refresher_info": refresher_info,
        "global_paused": global_paused
    })

@app.post("/api/global-pause")
async def api_global_pause(request: Request):
    b = await request.json()
    is_paused = bool(b.get("paused", False))
    supabase.rpc("set_global_pause", {"p_paused": is_paused}).execute()
    return JSONResponse({"ok": True, "paused": is_paused})

@app.post("/api/create-task")
async def api_create_task(request: Request):
    try:
        b = await request.json()
        start_id = str(int(b["start_id"])).zfill(10)
        end_id   = str(int(b["end_id"])).zfill(10)
        priority = int(b.get("priority", 100))

        if start_id[:3] != end_id[:3]:
            return JSONResponse({"ok": False, "error": "First 3 digits must match (year+series)."}, status_code=400)

        year = f"20{start_id[:2]}"
        series_digit = start_id[2]
        series_label = SERIES_MAP.get(series_digit, series_digit)
        series_name = f"{series_digit} - {series_label}"

        supabase.table("main_tasks").insert({
            "target_url_type": "eKhanij",
            "series_name":     series_name,
            "start_etp_id":    int(start_id),
            "end_etp_id":      int(end_id),
            "chunk_size":      int(b.get("chunk_size", 5000)),
            "concurrency":     int(b.get("concurrency", 300)),
            "priority":        priority
        }).execute()
        return JSONResponse({"ok": True})
    except Exception as e:
        return JSONResponse({"ok": False, "error": str(e)}, status_code=400)

@app.post("/api/tasks/{task_id}/action")
async def api_task_action(task_id: str, request: Request):
    b = await request.json()
    action = b.get("action")
    try:
        if action == "pause":
            supabase.rpc("pause_main_task", {"p_main_id": task_id}).execute()
        elif action == "resume":
            supabase.rpc("resume_main_task", {"p_main_id": task_id}).execute()
        elif action == "delete":
            supabase.table("main_tasks").delete().eq("id", task_id).execute()
        elif action == "priority":
            supabase.rpc("reorder_task_priority", {"p_main_id": task_id, "p_new_priority": int(b.get("priority", 100))}).execute()
        elif action == "retry":
            supabase.table("sub_tasks").update({"status": "pending", "assigned_worker_id": None})\
                .eq("status", "failed").eq("main_task_id", task_id).execute()
        return JSONResponse({"ok": True})
    except Exception as e:
        return JSONResponse({"ok": False, "error": str(e)}, status_code=400)

@app.post("/api/surgeon")
def api_surgeon():
    res = supabase.rpc("surgeon_recover_dead_tasks", {"p_timeout_minutes": 5}).execute()
    return JSONResponse({"ok": True, "recovered": res.data or 0})

# ─── FRONTEND HTML (PREMIUM UI — ITEM-LEVEL STATS + REFRESHER LOCK) ────────
HTML_CONTENT = r"""<!DOCTYPE html>
<html lang="en">
<head>
<meta charset="UTF-8"><meta name="viewport" content="width=device-width,initial-scale=1">
<title>ETP Command Center</title>
<link href="https://fonts.googleapis.com/css2?family=Inter:wght@400;500;600;700&display=swap" rel="stylesheet">
<style>
:root {
  --bg: #0f1115; --surface: #181b21; --surface-hover: #1f232b; --border: #2b303b;
  --text: #e2e8f0; --text-muted: #94a3b8;
  --blue: #3b82f6; --green: #10b981; --red: #ef4444; --yellow: #f59e0b; --purple: #8b5cf6; --cyan: #06b6d4;
}
* { box-sizing: border-box; margin: 0; padding: 0; font-family: 'Inter', sans-serif; }
body { background: var(--bg); color: var(--text); min-height: 100vh; }

header { background: var(--surface); border-bottom: 1px solid var(--border); padding: 15px 30px; display: flex; justify-content: space-between; align-items: center; position: sticky; top: 0; z-index: 100;}
h1 { font-size: 20px; font-weight: 700; display: flex; align-items: center; gap: 10px; }

.tabs { display: flex; background: var(--surface); border-bottom: 1px solid var(--border); padding: 0 30px; }
.tab { padding: 15px 20px; cursor: pointer; color: var(--text-muted); border-bottom: 2px solid transparent; font-weight: 500; font-size: 14px; transition: 0.2s;}
.tab:hover { color: var(--text); }
.tab.active { color: var(--blue); border-bottom-color: var(--blue); }

.content { padding: 30px; max-width: 1400px; margin: 0 auto; }
.panel { display: none; animation: fadeIn 0.3s ease; } .panel.active { display: block; }
@keyframes fadeIn { from { opacity: 0; transform: translateY(5px); } to { opacity: 1; transform: translateY(0); } }

/* Stats Grid */
.grid-stats { display: grid; grid-template-columns: repeat(auto-fit, minmax(160px, 1fr)); gap: 15px; margin-bottom: 25px; }
.stat-card { background: linear-gradient(180deg, var(--surface) 0%, var(--bg) 100%); border: 1px solid var(--border); border-radius: 12px; padding: 25px; display: flex; flex-direction: column; justify-content: center; position: relative; overflow: hidden;}
.stat-card::before { content: ''; position: absolute; top: 0; left: 0; width: 100%; height: 3px; background: var(--c); }
.stat-val { font-size: 28px; font-weight: 700; margin-bottom: 5px; color: var(--c); line-height: 1;}
.stat-lbl { font-size: 11px; color: var(--text-muted); font-weight: 500; text-transform: uppercase; letter-spacing: 0.5px; }

/* Cards & Tables */
.card { background: var(--surface); border: 1px solid var(--border); border-radius: 12px; padding: 20px; margin-bottom: 20px; box-shadow: 0 4px 6px rgba(0,0,0,0.1);}
.card-header { display: flex; justify-content: space-between; align-items: center; margin-bottom: 20px; padding-bottom: 15px; border-bottom: 1px solid var(--border);}
.card-header h3 { font-size: 16px; font-weight: 600; display: flex; align-items: center; gap: 8px;}

table { width: 100%; border-collapse: collapse; font-size: 14px; text-align: left; }
th { padding: 12px 15px; border-bottom: 1px solid var(--border); color: var(--text-muted); font-weight: 600; text-transform: uppercase; font-size: 12px; letter-spacing: 0.5px;}
td { padding: 15px; border-bottom: 1px solid var(--border); vertical-align: middle; }
tr:hover td { background: var(--surface-hover); }

/* Badges */
.badge { padding: 4px 10px; border-radius: 6px; font-size: 12px; font-weight: 600; border: 1px solid transparent; display: inline-block;}
.b-pending { background: rgba(245, 158, 11, 0.1); color: var(--yellow); border-color: rgba(245, 158, 11, 0.2); }
.b-running, .b-in_progress, .b-working { background: rgba(59, 130, 246, 0.1); color: var(--blue); border-color: rgba(59, 130, 246, 0.2); }
.b-completed, .b-idle { background: rgba(16, 185, 129, 0.1); color: var(--green); border-color: rgba(16, 185, 129, 0.2); }
.b-paused { background: rgba(148, 163, 184, 0.1); color: var(--text-muted); border-color: rgba(148, 163, 184, 0.2); }
.b-failed { background: rgba(239, 68, 68, 0.1); color: var(--red); border-color: rgba(239, 68, 68, 0.2); }

/* Progress Bar */
.prog-container { margin-top: 8px; }
.prog-stats { display: flex; justify-content: space-between; font-size: 12px; margin-bottom: 5px; color: var(--text-muted); font-weight: 500;}
.prog-bar { height: 8px; background: var(--bg); border-radius: 4px; overflow: hidden; border: 1px solid var(--border); display: flex;}
.prog-fill { height: 100%; transition: width 0.5s ease; }
.fill-green { background: var(--green); }
.fill-blue { background: var(--blue); }
.fill-red { background: var(--red); }
.fill-yellow { background: var(--yellow); }

/* Buttons */
button { background: var(--surface); color: var(--text); border: 1px solid var(--border); padding: 8px 14px; border-radius: 6px; cursor: pointer; font-size: 13px; font-weight: 500; transition: all 0.2s; display: inline-flex; align-items: center; gap: 6px;}
button:hover { background: var(--surface-hover); border-color: #475569; }
.btn-primary { background: var(--blue); border-color: var(--blue); color: white; }
.btn-primary:hover { background: #2563eb; border-color: #2563eb; }
.btn-danger { background: rgba(239, 68, 68, 0.1); color: var(--red); border-color: rgba(239, 68, 68, 0.3); }
.btn-danger:hover { background: var(--red); color: white; }

/* Forms & Inputs */
input { width: 100%; padding: 12px; background: var(--bg); border: 1px solid var(--border); color: var(--text); border-radius: 8px; margin-bottom: 15px; font-size: 14px; outline: none; transition: 0.2s;}
input:focus { border-color: var(--blue); box-shadow: 0 0 0 2px rgba(59, 130, 246, 0.2); }
label { display: block; margin-bottom: 6px; font-size: 13px; color: var(--text-muted); font-weight: 500;}
.form-row { display: grid; grid-template-columns: 1fr 1fr; gap: 20px; }

/* Global Switch */
.global-switch { display: flex; align-items: center; gap: 12px; background: var(--bg); padding: 8px 16px; border-radius: 8px; border: 1px solid var(--border); }
.pulse { width: 10px; height: 10px; border-radius: 50%; background: var(--green); box-shadow: 0 0 10px var(--green); animation: blink 2s infinite; }
.pulse.paused { background: var(--red); box-shadow: 0 0 10px var(--red); animation: none; }
@keyframes blink { 0%, 100% { opacity: 1; } 50% { opacity: 0.4; } }

/* Hash & Refresher Status */
.status-badge { display: flex; align-items: center; gap: 8px; background: var(--bg); padding: 6px 14px; border-radius: 8px; border: 1px solid var(--border); font-size: 12px; }
.status-dot { width: 8px; height: 8px; border-radius: 50%; }
.dot-active { background: var(--green); box-shadow: 0 0 8px var(--green); }
.dot-inactive { background: var(--red); box-shadow: 0 0 8px var(--red); }
.dot-warning { background: var(--yellow); box-shadow: 0 0 8px var(--yellow); }

.code-block { background: #0b0d10; padding: 20px; border-radius: 8px; font-family: 'Courier New', monospace; font-size: 13px; color: #a5d6ff; white-space: pre-wrap; border: 1px solid var(--border); overflow-x: auto; line-height: 1.5;}

/* Item count styling */
.item-count { font-variant-numeric: tabular-nums; font-weight: 600; }

/* Verification badge */
.verify-badge { display: inline-flex; align-items: center; gap: 4px; padding: 2px 8px; border-radius: 4px; font-size: 10px; font-weight: 600; }
.verify-ok { background: rgba(16, 185, 129, 0.15); color: var(--green); }
.verify-fail { background: rgba(239, 68, 68, 0.15); color: var(--red); }
</style>
</head>
<body>

<header>
  <h1><span style="font-size:24px">&#9874;</span> ETP Control Plane</h1>
  <div style="display:flex; align-items:center; gap: 12px;">
    <div class="status-badge" id="refresher-status">
      <div class="status-dot dot-inactive" id="refresher-dot"></div>
      <span id="refresher-text" style="color:var(--text-muted)">Refresher: Loading...</span>
    </div>
    <div class="status-badge" id="hash-status">
      <div class="status-dot dot-inactive" id="hash-dot"></div>
      <span id="hash-text" style="color:var(--text-muted)">Hash: Loading...</span>
    </div>
    <div style="font-size: 12px; color: var(--text-muted);" id="last-sync">Syncing...</div>
    <div class="global-switch">
      <div class="pulse" id="g-pulse"></div>
      <span id="g-status" style="font-weight:600; font-size: 14px;">SYSTEM ACTIVE</span>
      <button id="g-btn" onclick="toggleGlobalPause()"></button>
    </div>
  </div>
</header>

<div class="tabs">
  <div class="tab active" onclick="showTab('dashboard')">&#128202; Live Dashboard</div>
  <div class="tab" onclick="showTab('tasks')">&#10133; Manage Tasks</div>
  <div class="tab" onclick="showTab('workers')">&#129302; Launch Workers</div>
</div>

<div class="content">
  <!-- DASHBOARD -->
  <div id="dashboard" class="panel active">

    <div class="grid-stats">
      <div class="stat-card" style="--c: var(--green)"><div class="stat-val" id="st-prog">0%</div><div class="stat-lbl">Global Progress</div></div>
      <div class="stat-card" style="--c: var(--blue)"><div class="stat-val" id="st-workers">0</div><div class="stat-lbl">Active Workers</div></div>
      <div class="stat-card" style="--c: var(--purple)"><div class="stat-val" id="st-items">0</div><div class="stat-lbl">Total Items</div></div>
      <div class="stat-card" style="--c: var(--green)"><div class="stat-val" id="st-done">0</div><div class="stat-lbl">Done (Success)</div></div>
      <div class="stat-card" style="--c: var(--cyan)"><div class="stat-val" id="st-active">0</div><div class="stat-lbl">In Progress</div></div>
      <div class="stat-card" style="--c: var(--red)"><div class="stat-val" id="st-failed">0</div><div class="stat-lbl">Failed Items</div></div>
    </div>

    <!-- Math Verification Banner -->
    <div id="math-verify" style="background:var(--surface); border:1px solid var(--border); border-radius:8px; padding:10px 20px; margin-bottom:20px; display:flex; align-items:center; gap:15px; font-size:13px;">
      <span style="color:var(--text-muted)">Integrity Check:</span>
      <span id="verify-done" class="verify-badge verify-ok">Done: 0</span>
      <span style="color:var(--text-muted)">+</span>
      <span id="verify-pending" class="verify-badge" style="background:rgba(245,158,11,0.15);color:var(--yellow)">Pending: 0</span>
      <span style="color:var(--text-muted)">+</span>
      <span id="verify-active" class="verify-badge" style="background:rgba(59,130,246,0.15);color:var(--blue)">Active: 0</span>
      <span style="color:var(--text-muted)">=</span>
      <span id="verify-total" class="verify-badge verify-ok">Total: 0</span>
      <span id="verify-result" style="margin-left:auto;"></span>
    </div>

    <div class="card">
      <div class="card-header">
        <h3>&#128203; Task Priority Queue (Item-Level)</h3>
      </div>
      <div id="tasks-table">
        <p style="text-align:center; padding: 20px; color: var(--text-muted);">Loading system data...</p>
      </div>
    </div>

    <div class="card">
      <div class="card-header">
        <h3>&#129302; Live Worker Fleet</h3>
        <button onclick="runSurgeon()">&#127973; Force Clean Dead Workers</button>
      </div>
      <div id="workers-table">
        <p style="text-align:center; padding: 20px; color: var(--text-muted);">Loading workers...</p>
      </div>
    </div>
  </div>

  <!-- MANAGE TASKS -->
  <div id="tasks" class="panel">
    <div class="card" style="max-width: 800px; margin: 0 auto;">
      <div class="card-header">
        <h3>&#10133; Enqueue New Task</h3>
      </div>
      <p style="color:var(--text-muted); margin-bottom:20px; font-size:14px; line-height:1.5;">
        Tasks process strictly in order of Priority (Priority 1 processes before Priority 100).<br>
        If a chunk was already processed in the past, workers will automatically skip it (Delta Recovery).
      </p>

      <div class="form-row">
        <div><label>Start ETP ID</label><input type="number" id="t-start" placeholder="e.g. 2310000001"></div>
        <div><label>End ETP ID</label><input type="number" id="t-end" placeholder="e.g. 2310010000"></div>
      </div>
      <div class="form-row">
        <div><label>Priority Queue Number</label><input type="number" id="t-prio" value="100"></div>
        <div><label>Chunk Size</label><input type="number" id="t-chunk" value="5000"></div>
      </div>
      <div style="margin-top: 10px;">
        <button class="btn-primary" onclick="createTask()">&#128640; Add to Queue</button>
      </div>
    </div>
  </div>

  <!-- LAUNCH WORKERS -->
  <div id="workers" class="panel">

    <div class="card">
      <div class="card-header">
        <h3>&#128274; Hash Refresher (Run ONE Instance in Colab)</h3>
      </div>
      <p style="color:var(--text-muted); margin-bottom:20px; font-size:14px;">
        This MUST be running before workers can operate. It refreshes the auth_hash every 5 minutes and syncs the Data Lake.
        <br><strong>Distributed Lock:</strong> If you accidentally start multiple instances, only the LATEST one runs — all older ones self-terminate.
      </p>
      <div class="code-block"># 1. Install required packages silently
!pip install -q supabase requests huggingface_hub urllib3 lxml

import os, urllib.request

# 2. Pipeline Credentials
os.environ["SUPABASE_URL"] = "YOUR_SUPABASE_URL"
os.environ["SUPABASE_SERVICE_KEY"] = "YOUR_SUPABASE_KEY"
os.environ["HF_TOKEN"] = "YOUR_HF_TOKEN"
os.environ["HF_REPO_ID"] = "YOUR_REPO"

# 3. Download and run hash refresher
REPO_NAME = "YOUR_REPO_NAME"
urllib.request.urlretrieve(f"https://raw.githubusercontent.com/YOUR_GITHUB/{REPO_NAME}/main/hash_refresher.py", "hash_refresher.py")
!python hash_refresher.py</div>
    </div>

    <div class="card">
      <div class="card-header">
        <h3>&#127758; Launch Worker Nodes (Run as Many as You Want)</h3>
      </div>
      <p style="color:var(--text-muted); margin-bottom:20px; font-size:14px;">
        Workers are now thin and stateless. They read the auth_hash from the database and perform 1-step GETs. No more heavy boot sync. No more race conditions. Launch as many as you want simultaneously.
      </p>
      <div class="code-block"># 1. Install required packages silently
!pip install -q supabase requests beautifulsoup4 huggingface_hub urllib3 lxml

import os, urllib.request

# 2. Pipeline Credentials
os.environ["SUPABASE_URL"] = "YOUR_SUPABASE_URL"
os.environ["SUPABASE_SERVICE_KEY"] = "YOUR_SUPABASE_KEY"
os.environ["HF_TOKEN"] = "YOUR_HF_TOKEN"
os.environ["HF_REPO_ID"] = "YOUR_REPO"

# 3. Download and run thin worker
REPO_NAME = "YOUR_REPO_NAME"
urllib.request.urlretrieve(f"https://raw.githubusercontent.com/YOUR_GITHUB/{REPO_NAME}/main/worker.py", "worker.py")
!python worker.py</div>
    </div>
  </div>
</div>

<script>
let globalPaused = false;

function showTab(id){
  document.querySelectorAll('.panel').forEach(p=>p.classList.remove('active'));
  document.querySelectorAll('.tab').forEach(t=>t.classList.remove('active'));
  document.getElementById(id).classList.add('active');
  event.currentTarget.classList.add('active');
}

function timeAgo(dateString) {
    const seconds = Math.floor((new Date() - new Date(dateString)) / 1000);
    if(seconds < 60) return seconds + "s";
    const minutes = Math.floor(seconds / 60);
    if(minutes < 60) return minutes + "m " + (seconds % 60) + "s";
    const hours = Math.floor(minutes / 60);
    return hours + "h " + (minutes % 60) + "m";
}

function fmt(n) {
    return n.toLocaleString();
}

function loadData() {
  fetch('/api/dashboard').then(r=>r.json()).then(d=>{

    // Update Sync Timer
    let dte = new Date();
    document.getElementById('last-sync').innerText = "Live: " + dte.toLocaleTimeString();

    // Global Status
    globalPaused = d.global_paused;
    document.getElementById('g-pulse').className = globalPaused ? 'pulse paused' : 'pulse';
    document.getElementById('g-status').innerText = globalPaused ? 'FLEET PAUSED' : 'SYSTEM ACTIVE';
    document.getElementById('g-status').style.color = globalPaused ? 'var(--red)' : 'var(--green)';
    document.getElementById('g-btn').innerHTML = globalPaused ? '&#9654; Resume Fleet' : '&#9208; Pause Fleet';

    // Hash Status
    if (d.hash_info) {
        document.getElementById('hash-dot').className = 'status-dot dot-active';
        document.getElementById('hash-text').innerText = 'Hash: Active (' + timeAgo(d.hash_info.updated_at) + ' ago)';
        document.getElementById('hash-text').style.color = 'var(--green)';
    } else {
        document.getElementById('hash-dot').className = 'status-dot dot-inactive';
        document.getElementById('hash-text').innerText = 'Hash: MISSING - Run Refresher!';
        document.getElementById('hash-text').style.color = 'var(--red)';
    }

    // Refresher Lock Status
    if (d.refresher_info) {
        let hbAge = d.refresher_info.last_heartbeat ? timeAgo(d.refresher_info.last_heartbeat) : 'Unknown';
        let isStale = d.refresher_info.last_heartbeat && (new Date() - new Date(d.refresher_info.last_heartbeat)) > 600000; // 10 min stale
        if (isStale) {
            document.getElementById('refresher-dot').className = 'status-dot dot-warning';
            document.getElementById('refresher-text').innerText = 'Refresher: STALE (' + d.refresher_info.instance_id.substring(0,18) + '... last seen ' + hbAge + ' ago)';
            document.getElementById('refresher-text').style.color = 'var(--yellow)';
        } else {
            document.getElementById('refresher-dot').className = 'status-dot dot-active';
            document.getElementById('refresher-text').innerText = 'Refresher: Active [' + d.refresher_info.instance_id.substring(0,18) + '] ' + hbAge + ' ago';
            document.getElementById('refresher-text').style.color = 'var(--green)';
        }
    } else {
        document.getElementById('refresher-dot').className = 'status-dot dot-inactive';
        document.getElementById('refresher-text').innerText = 'Refresher: OFFLINE - Start Hash Refresher!';
        document.getElementById('refresher-text').style.color = 'var(--red)';
    }

    // Stats (ITEM-LEVEL, mathematically perfect)
    let gs = d.global_stats;
    let totalItems = gs.total_items || 1;
    let doneItems = (gs.success_items || 0) + (gs.failed_items || 0);
    let pct = ((gs.success_items / totalItems) * 100).toFixed(2);

    document.getElementById('st-prog').innerText = pct + '%';
    document.getElementById('st-items').innerText = fmt(gs.total_items);
    document.getElementById('st-done').innerText = fmt(gs.success_items);
    document.getElementById('st-active').innerText = fmt(gs.in_progress_items);
    document.getElementById('st-failed').innerText = fmt(gs.failed_items);
    document.getElementById('st-workers').innerText = d.workers.length;

    // Math Verification: done + pending + active = total
    let verifyDone = (gs.success_items || 0) + (gs.failed_items || 0);
    let verifyPending = gs.pending_items || 0;
    let verifyActive = gs.in_progress_items || 0;
    let verifyTotal = gs.total_items || 0;
    let verifySum = verifyDone + verifyPending + verifyActive;

    document.getElementById('verify-done').innerText = 'Done: ' + fmt(verifyDone);
    document.getElementById('verify-pending').innerText = 'Pending: ' + fmt(verifyPending);
    document.getElementById('verify-active').innerText = 'Active: ' + fmt(verifyActive);
    document.getElementById('verify-total').innerText = 'Total: ' + fmt(verifyTotal);

    if (verifySum === verifyTotal) {
        document.getElementById('verify-result').innerHTML = '<span class="verify-badge verify-ok">&#10003; Balanced</span>';
    } else {
        document.getElementById('verify-result').innerHTML = '<span class="verify-badge verify-fail">&#10007; Mismatch (sum=' + fmt(verifySum) + ')</span>';
    }

    // Tasks Table (ITEM-LEVEL PROGRESS)
    if(d.tasks.length === 0){
        document.getElementById('tasks-table').innerHTML = `<p style="text-align:center; padding:20px; color:var(--text-muted);">No tasks in queue. Go to Manage Tasks to create one.</p>`;
    } else {
        let thtml = `<table><tr><th width="80">Priority</th><th>Series</th><th width="40%">Item-Level Progress</th><th>Status</th><th width="150">Actions</th></tr>`;
        d.tasks.forEach(t=>{
          let is = t.item_stats;
          let ttot = is.total_items || 1;
          let pctDone = (is.success_items/ttot)*100;
          let pctProg = (is.in_progress_items/ttot)*100;
          let pctFail = (is.failed_items/ttot)*100;
          let pctPend = (is.pending_items/ttot)*100;

          let actions = '';
          if(t.status==='paused') actions += `<button onclick="actTask('${t.id}','resume')">&#9654; Resume</button> `;
          else if(t.status==='pending'||t.status==='running') actions += `<button onclick="actTask('${t.id}','pause')">&#9208; Pause</button> `;

          let retryBtn = (is.failed_items > 0 || is.pending_items > 0) ? `<button onclick="actTask('${t.id}','retry')" style="margin-top:5px; width:100%">&#128260; Retry ${fmt(is.failed_items)} Failed</button>` : '';

          thtml += `<tr>
            <td><span style="font-size: 16px; font-weight:700; color:var(--text)">#${t.priority}</span></td>
            <td>
                <span style="font-weight:600">${t.series_name}</span><br>
                <code style="color:var(--text-muted); font-size:12px;">${t.start_etp_id} &rarr; ${t.end_etp_id}</code>
            </td>
            <td>
                <div class="prog-stats">
                    <span class="item-count" style="color:var(--green)">${fmt(is.success_items)} Done</span>
                    <span class="item-count" style="color:var(--blue)">${fmt(is.in_progress_items)} Active</span>
                    <span class="item-count" style="color:var(--yellow)">${fmt(is.pending_items)} Pending</span>
                    <span class="item-count" style="color:var(--red)">${fmt(is.failed_items)} Failed</span>
                </div>
                <div class="prog-bar">
                    <div class="prog-fill fill-green" style="width:${pctDone}%"></div>
                    <div class="prog-fill fill-blue" style="width:${pctProg}%"></div>
                    <div class="prog-fill fill-red" style="width:${pctFail}%"></div>
                    <div class="prog-fill fill-yellow" style="width:${pctPend}%"></div>
                </div>
                <div style="margin-top:4px; font-size:11px; color:var(--text-muted);">
                    ${((is.success_items/ttot)*100).toFixed(2)}% of ${fmt(ttot)} items
                    ${is.failed_items > 0 ? ` | <span style="color:var(--red)">${fmt(is.failed_items)} need retry</span>` : ''}
                </div>
            </td>
            <td><span class="badge b-${t.status}">${t.status.toUpperCase()}</span></td>
            <td>
                <div style="display:flex; gap:5px;">
                    ${actions}
                    <button class="btn-danger" onclick="actTask('${t.id}','delete')" title="Delete Task">&#128465;</button>
                </div>
                ${retryBtn}
            </td>
          </tr>`;
        });
        document.getElementById('tasks-table').innerHTML = thtml + `</table>`;
    }

    // Workers Table
    if(d.workers.length === 0){
        document.getElementById('workers-table').innerHTML = `<p style="text-align:center; padding: 30px; color: var(--text-muted);">No active workers found.<br><br><span style="font-size:24px;">&#128564;</span></p>`;
    } else {
        let whtml = `<table><tr><th>Worker ID</th><th>Platform</th><th>Status</th><th>Total Uptime</th><th>Last Heartbeat</th></tr>`;
        d.workers.forEach(w=>{
            let activeColor = w.seconds_dead < 60 ? "var(--green)" : "var(--yellow)";

            whtml += `<tr>
                <td><code style="color:var(--blue); background:rgba(59,130,246,0.1); padding:4px 8px; border-radius:4px;">${w.id}</code></td>
                <td><span style="text-transform:capitalize; font-weight:500;">${w.platform}</span></td>
                <td><span class="badge b-${w.status}">${w.status.toUpperCase()}</span></td>
                <td style="font-weight:600">${timeAgo(w.started_at)}</td>
                <td>
                    <div style="display:flex; align-items:center; gap:8px;">
                        <div style="width:8px; height:8px; border-radius:50%; background:${activeColor};"></div>
                        <span style="color:var(--text-muted)">${w.seconds_dead}s ago</span>
                    </div>
                </td>
            </tr>`;
        });
        document.getElementById('workers-table').innerHTML = whtml + `</table>`;
    }
  }).catch(e => {
      document.getElementById('last-sync').innerText = "Connection lost. Retrying...";
  });
}

function actTask(id, action) {
  if(action==='delete' && !confirm('WARNING: Are you sure you want to completely delete this task?')) return;
  fetch(`/api/tasks/${id}/action`, {method:'POST', body:JSON.stringify({action})}).then(()=>loadData());
}

function toggleGlobalPause() {
  fetch('/api/global-pause', {method:'POST', body:JSON.stringify({paused: !globalPaused})}).then(()=>loadData());
}

function createTask() {
  let body = {
    start_id: document.getElementById('t-start').value,
    end_id: document.getElementById('t-end').value,
    priority: document.getElementById('t-prio').value,
    chunk_size: document.getElementById('t-chunk').value
  };

  if(!body.start_id || !body.end_id) { alert("Please enter both Start and End IDs."); return; }

  fetch('/api/create-task', {method:'POST', body:JSON.stringify(body)}).then(r=>r.json()).then(d=>{
    if(d.ok){ alert('Task successfully added to the Queue!'); loadData(); showTab('dashboard'); }
    else alert('Error: ' + d.error);
  });
}

function runSurgeon() {
  if(!confirm("This will force-clean any dead workers and recover stuck chunks immediately. Continue?")) return;
  fetch('/api/surgeon', {method:'POST'}).then(r=>r.json()).then(d=>{
    alert('Surgeon finished. Recovered chunks: ' + d.recovered); loadData();
  });
}

// ── FAST POLLING (Every 3 seconds) ──
setInterval(loadData, 3000);
loadData();
</script>
</body>
</html>
"""

@app.get("/", response_class=HTMLResponse)
def root():
    return HTML_CONTENT

if __name__ == "__main__":
    uvicorn.run(app, host="0.0.0.0", port=7860)