Spaces:
Sleeping
Sleeping
| """ | |
| app.py — ETP Distributed Scraper: Control Plane v5.1 (Item-Level Dashboard + Hash Status + Refresher Lock) | |
| FastAPI + HTML/JS. | |
| KEY CHANGES FROM v5: | |
| 1. DASHBOARD: Uses fixed get_dashboard_item_stats() RPC | |
| - success_items and failed_items counted for ALL chunk statuses (not just completed/failed) | |
| - This ensures: done + pending = total ALWAYS HOLDS | |
| - No more 1,000-row .select() limit | |
| - Mathematically perfect item-level progress | |
| 2. HASH STATUS: Shows current auth_hash status from system_tokens | |
| - When it was last refreshed | |
| - Whether it's active | |
| 3. REFRESHER LOCK STATUS: Shows which hash_refresher instance is active | |
| - Instance ID | |
| - Last heartbeat time | |
| - If no refresher is running, shows warning | |
| 4. Removed direct sub_tasks .select() — all stats via RPC | |
| """ | |
| import os, time, threading, logging | |
| from datetime import datetime, timezone | |
| from supabase import create_client, Client | |
| from fastapi import FastAPI, Request | |
| from fastapi.responses import HTMLResponse, JSONResponse | |
| import uvicorn | |
| logging.basicConfig(level=logging.INFO, format="%(asctime)s [%(levelname)s] %(message)s") | |
| log = logging.getLogger("ControlPlane") | |
| SUPABASE_URL = os.environ["SUPABASE_URL"] | |
| SUPABASE_KEY = os.environ["SUPABASE_SERVICE_KEY"] | |
| SURGEON_TIMEOUT = int(os.environ.get("SURGEON_TIMEOUT_MINUTES", "5")) | |
| WATCHDOG_INTERVAL = 300 | |
| supabase: Client = create_client(SUPABASE_URL, SUPABASE_KEY) | |
| app = FastAPI() | |
| SERIES_MAP = {"0": "QL", "1": "ST", "2": "ML"} | |
| # ─── BACKGROUND SURGEON (Auto-Cleaner) ────────────────────────────────── | |
| def watchdog_loop(): | |
| log.info("Watchdog started. Will permanently delete dead workers every 5 mins.") | |
| while True: | |
| try: | |
| res = supabase.rpc("surgeon_recover_dead_tasks", {"p_timeout_minutes": SURGEON_TIMEOUT}).execute() | |
| recovered = res.data or 0 | |
| if recovered > 0: | |
| log.info(f"Surgeon recovered {recovered} stuck chunks and wiped dead workers.") | |
| except Exception as e: | |
| log.error(f"Surgeon error: {e}") | |
| time.sleep(WATCHDOG_INTERVAL) | |
| threading.Thread(target=watchdog_loop, daemon=True).start() | |
| # ─── API ROUTES ──────────────────────────────────────────────── | |
| def api_dashboard(): | |
| # 1. Fetch tasks (few rows, no limit issue) | |
| tasks = supabase.table("main_tasks").select("*").order("priority").order("created_at").execute().data or [] | |
| # 2. Fetch item-level stats via RPC (server-side aggregation, no row limit) | |
| # Fixed RPC: success_items + failed_items counted for ALL statuses | |
| # This ensures: done + pending = total ALWAYS | |
| stats_data = supabase.rpc("get_dashboard_item_stats").execute().data or [] | |
| stats_map = {} | |
| for s in stats_data: | |
| stats_map[s["main_task_id"]] = s | |
| # 3. Compute global item-level stats | |
| global_stats = {"total_items": 0, "pending_items": 0, "in_progress_items": 0, "success_items": 0, "failed_items": 0} | |
| for t in tasks: | |
| s = stats_map.get(t["id"], {}) | |
| t["item_stats"] = { | |
| "total_items": s.get("total_items", 0) or 0, | |
| "success_items": s.get("success_items", 0) or 0, | |
| "failed_items": s.get("failed_items", 0) or 0, | |
| "in_progress_items": s.get("in_progress_items", 0) or 0, | |
| "pending_items": s.get("pending_items", 0) or 0, | |
| } | |
| for key in global_stats: | |
| global_stats[key] += t["item_stats"].get(key, 0) | |
| # 4. Fetch and filter live workers (HIDE IF DEAD > 5 MINS) | |
| raw_workers = supabase.table("workers").select("*").order("started_at", desc=True).execute().data or [] | |
| live_workers = [] | |
| now_utc = datetime.now(timezone.utc) | |
| for w in raw_workers: | |
| try: | |
| hb_str = w["last_heartbeat"].replace("Z", "+00:00") | |
| hb_time = datetime.fromisoformat(hb_str) | |
| seconds_dead = (now_utc - hb_time).total_seconds() | |
| if seconds_dead <= 300: | |
| w["seconds_dead"] = int(seconds_dead) | |
| live_workers.append(w) | |
| except Exception: | |
| pass | |
| # 5. Fetch System State | |
| sys_config = supabase.table("system_config").select("*").eq("id", 1).execute().data | |
| global_paused = sys_config[0]["is_globally_paused"] if sys_config else False | |
| # 6. Fetch Auth Hash Status | |
| hash_info = None | |
| try: | |
| hash_res = supabase.rpc("get_active_auth_hash").execute() | |
| if hash_res.data and len(hash_res.data) > 0: | |
| hash_info = hash_res.data[0] | |
| except Exception: | |
| pass | |
| # 7. Fetch Refresher Lock Status | |
| refresher_info = None | |
| if sys_config: | |
| sc = sys_config[0] | |
| if sc.get("active_refresher_id"): | |
| refresher_info = { | |
| "instance_id": sc["active_refresher_id"], | |
| "last_heartbeat": sc.get("refresher_last_heartbeat"), | |
| } | |
| return JSONResponse({ | |
| "tasks": tasks, | |
| "workers": live_workers, | |
| "global_stats": global_stats, | |
| "hash_info": hash_info, | |
| "refresher_info": refresher_info, | |
| "global_paused": global_paused | |
| }) | |
| async def api_global_pause(request: Request): | |
| b = await request.json() | |
| is_paused = bool(b.get("paused", False)) | |
| supabase.rpc("set_global_pause", {"p_paused": is_paused}).execute() | |
| return JSONResponse({"ok": True, "paused": is_paused}) | |
| async def api_create_task(request: Request): | |
| try: | |
| b = await request.json() | |
| start_id = str(int(b["start_id"])).zfill(10) | |
| end_id = str(int(b["end_id"])).zfill(10) | |
| priority = int(b.get("priority", 100)) | |
| if start_id[:3] != end_id[:3]: | |
| return JSONResponse({"ok": False, "error": "First 3 digits must match (year+series)."}, status_code=400) | |
| year = f"20{start_id[:2]}" | |
| series_digit = start_id[2] | |
| series_label = SERIES_MAP.get(series_digit, series_digit) | |
| series_name = f"{series_digit} - {series_label}" | |
| supabase.table("main_tasks").insert({ | |
| "target_url_type": "eKhanij", | |
| "series_name": series_name, | |
| "start_etp_id": int(start_id), | |
| "end_etp_id": int(end_id), | |
| "chunk_size": int(b.get("chunk_size", 5000)), | |
| "concurrency": int(b.get("concurrency", 300)), | |
| "priority": priority | |
| }).execute() | |
| return JSONResponse({"ok": True}) | |
| except Exception as e: | |
| return JSONResponse({"ok": False, "error": str(e)}, status_code=400) | |
| async def api_task_action(task_id: str, request: Request): | |
| b = await request.json() | |
| action = b.get("action") | |
| try: | |
| if action == "pause": | |
| supabase.rpc("pause_main_task", {"p_main_id": task_id}).execute() | |
| elif action == "resume": | |
| supabase.rpc("resume_main_task", {"p_main_id": task_id}).execute() | |
| elif action == "delete": | |
| supabase.table("main_tasks").delete().eq("id", task_id).execute() | |
| elif action == "priority": | |
| supabase.rpc("reorder_task_priority", {"p_main_id": task_id, "p_new_priority": int(b.get("priority", 100))}).execute() | |
| elif action == "retry": | |
| supabase.table("sub_tasks").update({"status": "pending", "assigned_worker_id": None})\ | |
| .eq("status", "failed").eq("main_task_id", task_id).execute() | |
| return JSONResponse({"ok": True}) | |
| except Exception as e: | |
| return JSONResponse({"ok": False, "error": str(e)}, status_code=400) | |
| def api_surgeon(): | |
| res = supabase.rpc("surgeon_recover_dead_tasks", {"p_timeout_minutes": 5}).execute() | |
| return JSONResponse({"ok": True, "recovered": res.data or 0}) | |
| # ─── FRONTEND HTML (PREMIUM UI — ITEM-LEVEL STATS + REFRESHER LOCK) ──────── | |
| HTML_CONTENT = r"""<!DOCTYPE html> | |
| <html lang="en"> | |
| <head> | |
| <meta charset="UTF-8"><meta name="viewport" content="width=device-width,initial-scale=1"> | |
| <title>ETP Command Center</title> | |
| <link href="https://fonts.googleapis.com/css2?family=Inter:wght@400;500;600;700&display=swap" rel="stylesheet"> | |
| <style> | |
| :root { | |
| --bg: #0f1115; --surface: #181b21; --surface-hover: #1f232b; --border: #2b303b; | |
| --text: #e2e8f0; --text-muted: #94a3b8; | |
| --blue: #3b82f6; --green: #10b981; --red: #ef4444; --yellow: #f59e0b; --purple: #8b5cf6; --cyan: #06b6d4; | |
| } | |
| * { box-sizing: border-box; margin: 0; padding: 0; font-family: 'Inter', sans-serif; } | |
| body { background: var(--bg); color: var(--text); min-height: 100vh; } | |
| header { background: var(--surface); border-bottom: 1px solid var(--border); padding: 15px 30px; display: flex; justify-content: space-between; align-items: center; position: sticky; top: 0; z-index: 100;} | |
| h1 { font-size: 20px; font-weight: 700; display: flex; align-items: center; gap: 10px; } | |
| .tabs { display: flex; background: var(--surface); border-bottom: 1px solid var(--border); padding: 0 30px; } | |
| .tab { padding: 15px 20px; cursor: pointer; color: var(--text-muted); border-bottom: 2px solid transparent; font-weight: 500; font-size: 14px; transition: 0.2s;} | |
| .tab:hover { color: var(--text); } | |
| .tab.active { color: var(--blue); border-bottom-color: var(--blue); } | |
| .content { padding: 30px; max-width: 1400px; margin: 0 auto; } | |
| .panel { display: none; animation: fadeIn 0.3s ease; } .panel.active { display: block; } | |
| @keyframes fadeIn { from { opacity: 0; transform: translateY(5px); } to { opacity: 1; transform: translateY(0); } } | |
| /* Stats Grid */ | |
| .grid-stats { display: grid; grid-template-columns: repeat(auto-fit, minmax(160px, 1fr)); gap: 15px; margin-bottom: 25px; } | |
| .stat-card { background: linear-gradient(180deg, var(--surface) 0%, var(--bg) 100%); border: 1px solid var(--border); border-radius: 12px; padding: 25px; display: flex; flex-direction: column; justify-content: center; position: relative; overflow: hidden;} | |
| .stat-card::before { content: ''; position: absolute; top: 0; left: 0; width: 100%; height: 3px; background: var(--c); } | |
| .stat-val { font-size: 28px; font-weight: 700; margin-bottom: 5px; color: var(--c); line-height: 1;} | |
| .stat-lbl { font-size: 11px; color: var(--text-muted); font-weight: 500; text-transform: uppercase; letter-spacing: 0.5px; } | |
| /* Cards & Tables */ | |
| .card { background: var(--surface); border: 1px solid var(--border); border-radius: 12px; padding: 20px; margin-bottom: 20px; box-shadow: 0 4px 6px rgba(0,0,0,0.1);} | |
| .card-header { display: flex; justify-content: space-between; align-items: center; margin-bottom: 20px; padding-bottom: 15px; border-bottom: 1px solid var(--border);} | |
| .card-header h3 { font-size: 16px; font-weight: 600; display: flex; align-items: center; gap: 8px;} | |
| table { width: 100%; border-collapse: collapse; font-size: 14px; text-align: left; } | |
| th { padding: 12px 15px; border-bottom: 1px solid var(--border); color: var(--text-muted); font-weight: 600; text-transform: uppercase; font-size: 12px; letter-spacing: 0.5px;} | |
| td { padding: 15px; border-bottom: 1px solid var(--border); vertical-align: middle; } | |
| tr:hover td { background: var(--surface-hover); } | |
| /* Badges */ | |
| .badge { padding: 4px 10px; border-radius: 6px; font-size: 12px; font-weight: 600; border: 1px solid transparent; display: inline-block;} | |
| .b-pending { background: rgba(245, 158, 11, 0.1); color: var(--yellow); border-color: rgba(245, 158, 11, 0.2); } | |
| .b-running, .b-in_progress, .b-working { background: rgba(59, 130, 246, 0.1); color: var(--blue); border-color: rgba(59, 130, 246, 0.2); } | |
| .b-completed, .b-idle { background: rgba(16, 185, 129, 0.1); color: var(--green); border-color: rgba(16, 185, 129, 0.2); } | |
| .b-paused { background: rgba(148, 163, 184, 0.1); color: var(--text-muted); border-color: rgba(148, 163, 184, 0.2); } | |
| .b-failed { background: rgba(239, 68, 68, 0.1); color: var(--red); border-color: rgba(239, 68, 68, 0.2); } | |
| /* Progress Bar */ | |
| .prog-container { margin-top: 8px; } | |
| .prog-stats { display: flex; justify-content: space-between; font-size: 12px; margin-bottom: 5px; color: var(--text-muted); font-weight: 500;} | |
| .prog-bar { height: 8px; background: var(--bg); border-radius: 4px; overflow: hidden; border: 1px solid var(--border); display: flex;} | |
| .prog-fill { height: 100%; transition: width 0.5s ease; } | |
| .fill-green { background: var(--green); } | |
| .fill-blue { background: var(--blue); } | |
| .fill-red { background: var(--red); } | |
| .fill-yellow { background: var(--yellow); } | |
| /* Buttons */ | |
| button { background: var(--surface); color: var(--text); border: 1px solid var(--border); padding: 8px 14px; border-radius: 6px; cursor: pointer; font-size: 13px; font-weight: 500; transition: all 0.2s; display: inline-flex; align-items: center; gap: 6px;} | |
| button:hover { background: var(--surface-hover); border-color: #475569; } | |
| .btn-primary { background: var(--blue); border-color: var(--blue); color: white; } | |
| .btn-primary:hover { background: #2563eb; border-color: #2563eb; } | |
| .btn-danger { background: rgba(239, 68, 68, 0.1); color: var(--red); border-color: rgba(239, 68, 68, 0.3); } | |
| .btn-danger:hover { background: var(--red); color: white; } | |
| /* Forms & Inputs */ | |
| input { width: 100%; padding: 12px; background: var(--bg); border: 1px solid var(--border); color: var(--text); border-radius: 8px; margin-bottom: 15px; font-size: 14px; outline: none; transition: 0.2s;} | |
| input:focus { border-color: var(--blue); box-shadow: 0 0 0 2px rgba(59, 130, 246, 0.2); } | |
| label { display: block; margin-bottom: 6px; font-size: 13px; color: var(--text-muted); font-weight: 500;} | |
| .form-row { display: grid; grid-template-columns: 1fr 1fr; gap: 20px; } | |
| /* Global Switch */ | |
| .global-switch { display: flex; align-items: center; gap: 12px; background: var(--bg); padding: 8px 16px; border-radius: 8px; border: 1px solid var(--border); } | |
| .pulse { width: 10px; height: 10px; border-radius: 50%; background: var(--green); box-shadow: 0 0 10px var(--green); animation: blink 2s infinite; } | |
| .pulse.paused { background: var(--red); box-shadow: 0 0 10px var(--red); animation: none; } | |
| @keyframes blink { 0%, 100% { opacity: 1; } 50% { opacity: 0.4; } } | |
| /* Hash & Refresher Status */ | |
| .status-badge { display: flex; align-items: center; gap: 8px; background: var(--bg); padding: 6px 14px; border-radius: 8px; border: 1px solid var(--border); font-size: 12px; } | |
| .status-dot { width: 8px; height: 8px; border-radius: 50%; } | |
| .dot-active { background: var(--green); box-shadow: 0 0 8px var(--green); } | |
| .dot-inactive { background: var(--red); box-shadow: 0 0 8px var(--red); } | |
| .dot-warning { background: var(--yellow); box-shadow: 0 0 8px var(--yellow); } | |
| .code-block { background: #0b0d10; padding: 20px; border-radius: 8px; font-family: 'Courier New', monospace; font-size: 13px; color: #a5d6ff; white-space: pre-wrap; border: 1px solid var(--border); overflow-x: auto; line-height: 1.5;} | |
| /* Item count styling */ | |
| .item-count { font-variant-numeric: tabular-nums; font-weight: 600; } | |
| /* Verification badge */ | |
| .verify-badge { display: inline-flex; align-items: center; gap: 4px; padding: 2px 8px; border-radius: 4px; font-size: 10px; font-weight: 600; } | |
| .verify-ok { background: rgba(16, 185, 129, 0.15); color: var(--green); } | |
| .verify-fail { background: rgba(239, 68, 68, 0.15); color: var(--red); } | |
| </style> | |
| </head> | |
| <body> | |
| <header> | |
| <h1><span style="font-size:24px">⚒</span> ETP Control Plane</h1> | |
| <div style="display:flex; align-items:center; gap: 12px;"> | |
| <div class="status-badge" id="refresher-status"> | |
| <div class="status-dot dot-inactive" id="refresher-dot"></div> | |
| <span id="refresher-text" style="color:var(--text-muted)">Refresher: Loading...</span> | |
| </div> | |
| <div class="status-badge" id="hash-status"> | |
| <div class="status-dot dot-inactive" id="hash-dot"></div> | |
| <span id="hash-text" style="color:var(--text-muted)">Hash: Loading...</span> | |
| </div> | |
| <div style="font-size: 12px; color: var(--text-muted);" id="last-sync">Syncing...</div> | |
| <div class="global-switch"> | |
| <div class="pulse" id="g-pulse"></div> | |
| <span id="g-status" style="font-weight:600; font-size: 14px;">SYSTEM ACTIVE</span> | |
| <button id="g-btn" onclick="toggleGlobalPause()"></button> | |
| </div> | |
| </div> | |
| </header> | |
| <div class="tabs"> | |
| <div class="tab active" onclick="showTab('dashboard')">📊 Live Dashboard</div> | |
| <div class="tab" onclick="showTab('tasks')">➕ Manage Tasks</div> | |
| <div class="tab" onclick="showTab('workers')">🤖 Launch Workers</div> | |
| </div> | |
| <div class="content"> | |
| <!-- DASHBOARD --> | |
| <div id="dashboard" class="panel active"> | |
| <div class="grid-stats"> | |
| <div class="stat-card" style="--c: var(--green)"><div class="stat-val" id="st-prog">0%</div><div class="stat-lbl">Global Progress</div></div> | |
| <div class="stat-card" style="--c: var(--blue)"><div class="stat-val" id="st-workers">0</div><div class="stat-lbl">Active Workers</div></div> | |
| <div class="stat-card" style="--c: var(--purple)"><div class="stat-val" id="st-items">0</div><div class="stat-lbl">Total Items</div></div> | |
| <div class="stat-card" style="--c: var(--green)"><div class="stat-val" id="st-done">0</div><div class="stat-lbl">Done (Success)</div></div> | |
| <div class="stat-card" style="--c: var(--cyan)"><div class="stat-val" id="st-active">0</div><div class="stat-lbl">In Progress</div></div> | |
| <div class="stat-card" style="--c: var(--red)"><div class="stat-val" id="st-failed">0</div><div class="stat-lbl">Failed Items</div></div> | |
| </div> | |
| <!-- Math Verification Banner --> | |
| <div id="math-verify" style="background:var(--surface); border:1px solid var(--border); border-radius:8px; padding:10px 20px; margin-bottom:20px; display:flex; align-items:center; gap:15px; font-size:13px;"> | |
| <span style="color:var(--text-muted)">Integrity Check:</span> | |
| <span id="verify-done" class="verify-badge verify-ok">Done: 0</span> | |
| <span style="color:var(--text-muted)">+</span> | |
| <span id="verify-pending" class="verify-badge" style="background:rgba(245,158,11,0.15);color:var(--yellow)">Pending: 0</span> | |
| <span style="color:var(--text-muted)">+</span> | |
| <span id="verify-active" class="verify-badge" style="background:rgba(59,130,246,0.15);color:var(--blue)">Active: 0</span> | |
| <span style="color:var(--text-muted)">=</span> | |
| <span id="verify-total" class="verify-badge verify-ok">Total: 0</span> | |
| <span id="verify-result" style="margin-left:auto;"></span> | |
| </div> | |
| <div class="card"> | |
| <div class="card-header"> | |
| <h3>📋 Task Priority Queue (Item-Level)</h3> | |
| </div> | |
| <div id="tasks-table"> | |
| <p style="text-align:center; padding: 20px; color: var(--text-muted);">Loading system data...</p> | |
| </div> | |
| </div> | |
| <div class="card"> | |
| <div class="card-header"> | |
| <h3>🤖 Live Worker Fleet</h3> | |
| <button onclick="runSurgeon()">🏥 Force Clean Dead Workers</button> | |
| </div> | |
| <div id="workers-table"> | |
| <p style="text-align:center; padding: 20px; color: var(--text-muted);">Loading workers...</p> | |
| </div> | |
| </div> | |
| </div> | |
| <!-- MANAGE TASKS --> | |
| <div id="tasks" class="panel"> | |
| <div class="card" style="max-width: 800px; margin: 0 auto;"> | |
| <div class="card-header"> | |
| <h3>➕ Enqueue New Task</h3> | |
| </div> | |
| <p style="color:var(--text-muted); margin-bottom:20px; font-size:14px; line-height:1.5;"> | |
| Tasks process strictly in order of Priority (Priority 1 processes before Priority 100).<br> | |
| If a chunk was already processed in the past, workers will automatically skip it (Delta Recovery). | |
| </p> | |
| <div class="form-row"> | |
| <div><label>Start ETP ID</label><input type="number" id="t-start" placeholder="e.g. 2310000001"></div> | |
| <div><label>End ETP ID</label><input type="number" id="t-end" placeholder="e.g. 2310010000"></div> | |
| </div> | |
| <div class="form-row"> | |
| <div><label>Priority Queue Number</label><input type="number" id="t-prio" value="100"></div> | |
| <div><label>Chunk Size</label><input type="number" id="t-chunk" value="5000"></div> | |
| </div> | |
| <div style="margin-top: 10px;"> | |
| <button class="btn-primary" onclick="createTask()">🚀 Add to Queue</button> | |
| </div> | |
| </div> | |
| </div> | |
| <!-- LAUNCH WORKERS --> | |
| <div id="workers" class="panel"> | |
| <div class="card"> | |
| <div class="card-header"> | |
| <h3>🔒 Hash Refresher (Run ONE Instance in Colab)</h3> | |
| </div> | |
| <p style="color:var(--text-muted); margin-bottom:20px; font-size:14px;"> | |
| This MUST be running before workers can operate. It refreshes the auth_hash every 5 minutes and syncs the Data Lake. | |
| <br><strong>Distributed Lock:</strong> If you accidentally start multiple instances, only the LATEST one runs — all older ones self-terminate. | |
| </p> | |
| <div class="code-block"># 1. Install required packages silently | |
| !pip install -q supabase requests huggingface_hub urllib3 lxml | |
| import os, urllib.request | |
| # 2. Pipeline Credentials | |
| os.environ["SUPABASE_URL"] = "YOUR_SUPABASE_URL" | |
| os.environ["SUPABASE_SERVICE_KEY"] = "YOUR_SUPABASE_KEY" | |
| os.environ["HF_TOKEN"] = "YOUR_HF_TOKEN" | |
| os.environ["HF_REPO_ID"] = "YOUR_REPO" | |
| # 3. Download and run hash refresher | |
| REPO_NAME = "YOUR_REPO_NAME" | |
| urllib.request.urlretrieve(f"https://raw.githubusercontent.com/YOUR_GITHUB/{REPO_NAME}/main/hash_refresher.py", "hash_refresher.py") | |
| !python hash_refresher.py</div> | |
| </div> | |
| <div class="card"> | |
| <div class="card-header"> | |
| <h3>🌎 Launch Worker Nodes (Run as Many as You Want)</h3> | |
| </div> | |
| <p style="color:var(--text-muted); margin-bottom:20px; font-size:14px;"> | |
| Workers are now thin and stateless. They read the auth_hash from the database and perform 1-step GETs. No more heavy boot sync. No more race conditions. Launch as many as you want simultaneously. | |
| </p> | |
| <div class="code-block"># 1. Install required packages silently | |
| !pip install -q supabase requests beautifulsoup4 huggingface_hub urllib3 lxml | |
| import os, urllib.request | |
| # 2. Pipeline Credentials | |
| os.environ["SUPABASE_URL"] = "YOUR_SUPABASE_URL" | |
| os.environ["SUPABASE_SERVICE_KEY"] = "YOUR_SUPABASE_KEY" | |
| os.environ["HF_TOKEN"] = "YOUR_HF_TOKEN" | |
| os.environ["HF_REPO_ID"] = "YOUR_REPO" | |
| # 3. Download and run thin worker | |
| REPO_NAME = "YOUR_REPO_NAME" | |
| urllib.request.urlretrieve(f"https://raw.githubusercontent.com/YOUR_GITHUB/{REPO_NAME}/main/worker.py", "worker.py") | |
| !python worker.py</div> | |
| </div> | |
| </div> | |
| </div> | |
| <script> | |
| let globalPaused = false; | |
| function showTab(id){ | |
| document.querySelectorAll('.panel').forEach(p=>p.classList.remove('active')); | |
| document.querySelectorAll('.tab').forEach(t=>t.classList.remove('active')); | |
| document.getElementById(id).classList.add('active'); | |
| event.currentTarget.classList.add('active'); | |
| } | |
| function timeAgo(dateString) { | |
| const seconds = Math.floor((new Date() - new Date(dateString)) / 1000); | |
| if(seconds < 60) return seconds + "s"; | |
| const minutes = Math.floor(seconds / 60); | |
| if(minutes < 60) return minutes + "m " + (seconds % 60) + "s"; | |
| const hours = Math.floor(minutes / 60); | |
| return hours + "h " + (minutes % 60) + "m"; | |
| } | |
| function fmt(n) { | |
| return n.toLocaleString(); | |
| } | |
| function loadData() { | |
| fetch('/api/dashboard').then(r=>r.json()).then(d=>{ | |
| // Update Sync Timer | |
| let dte = new Date(); | |
| document.getElementById('last-sync').innerText = "Live: " + dte.toLocaleTimeString(); | |
| // Global Status | |
| globalPaused = d.global_paused; | |
| document.getElementById('g-pulse').className = globalPaused ? 'pulse paused' : 'pulse'; | |
| document.getElementById('g-status').innerText = globalPaused ? 'FLEET PAUSED' : 'SYSTEM ACTIVE'; | |
| document.getElementById('g-status').style.color = globalPaused ? 'var(--red)' : 'var(--green)'; | |
| document.getElementById('g-btn').innerHTML = globalPaused ? '▶ Resume Fleet' : '⏸ Pause Fleet'; | |
| // Hash Status | |
| if (d.hash_info) { | |
| document.getElementById('hash-dot').className = 'status-dot dot-active'; | |
| document.getElementById('hash-text').innerText = 'Hash: Active (' + timeAgo(d.hash_info.updated_at) + ' ago)'; | |
| document.getElementById('hash-text').style.color = 'var(--green)'; | |
| } else { | |
| document.getElementById('hash-dot').className = 'status-dot dot-inactive'; | |
| document.getElementById('hash-text').innerText = 'Hash: MISSING - Run Refresher!'; | |
| document.getElementById('hash-text').style.color = 'var(--red)'; | |
| } | |
| // Refresher Lock Status | |
| if (d.refresher_info) { | |
| let hbAge = d.refresher_info.last_heartbeat ? timeAgo(d.refresher_info.last_heartbeat) : 'Unknown'; | |
| let isStale = d.refresher_info.last_heartbeat && (new Date() - new Date(d.refresher_info.last_heartbeat)) > 600000; // 10 min stale | |
| if (isStale) { | |
| document.getElementById('refresher-dot').className = 'status-dot dot-warning'; | |
| document.getElementById('refresher-text').innerText = 'Refresher: STALE (' + d.refresher_info.instance_id.substring(0,18) + '... last seen ' + hbAge + ' ago)'; | |
| document.getElementById('refresher-text').style.color = 'var(--yellow)'; | |
| } else { | |
| document.getElementById('refresher-dot').className = 'status-dot dot-active'; | |
| document.getElementById('refresher-text').innerText = 'Refresher: Active [' + d.refresher_info.instance_id.substring(0,18) + '] ' + hbAge + ' ago'; | |
| document.getElementById('refresher-text').style.color = 'var(--green)'; | |
| } | |
| } else { | |
| document.getElementById('refresher-dot').className = 'status-dot dot-inactive'; | |
| document.getElementById('refresher-text').innerText = 'Refresher: OFFLINE - Start Hash Refresher!'; | |
| document.getElementById('refresher-text').style.color = 'var(--red)'; | |
| } | |
| // Stats (ITEM-LEVEL, mathematically perfect) | |
| let gs = d.global_stats; | |
| let totalItems = gs.total_items || 1; | |
| let doneItems = (gs.success_items || 0) + (gs.failed_items || 0); | |
| let pct = ((gs.success_items / totalItems) * 100).toFixed(2); | |
| document.getElementById('st-prog').innerText = pct + '%'; | |
| document.getElementById('st-items').innerText = fmt(gs.total_items); | |
| document.getElementById('st-done').innerText = fmt(gs.success_items); | |
| document.getElementById('st-active').innerText = fmt(gs.in_progress_items); | |
| document.getElementById('st-failed').innerText = fmt(gs.failed_items); | |
| document.getElementById('st-workers').innerText = d.workers.length; | |
| // Math Verification: done + pending + active = total | |
| let verifyDone = (gs.success_items || 0) + (gs.failed_items || 0); | |
| let verifyPending = gs.pending_items || 0; | |
| let verifyActive = gs.in_progress_items || 0; | |
| let verifyTotal = gs.total_items || 0; | |
| let verifySum = verifyDone + verifyPending + verifyActive; | |
| document.getElementById('verify-done').innerText = 'Done: ' + fmt(verifyDone); | |
| document.getElementById('verify-pending').innerText = 'Pending: ' + fmt(verifyPending); | |
| document.getElementById('verify-active').innerText = 'Active: ' + fmt(verifyActive); | |
| document.getElementById('verify-total').innerText = 'Total: ' + fmt(verifyTotal); | |
| if (verifySum === verifyTotal) { | |
| document.getElementById('verify-result').innerHTML = '<span class="verify-badge verify-ok">✓ Balanced</span>'; | |
| } else { | |
| document.getElementById('verify-result').innerHTML = '<span class="verify-badge verify-fail">✗ Mismatch (sum=' + fmt(verifySum) + ')</span>'; | |
| } | |
| // Tasks Table (ITEM-LEVEL PROGRESS) | |
| if(d.tasks.length === 0){ | |
| document.getElementById('tasks-table').innerHTML = `<p style="text-align:center; padding:20px; color:var(--text-muted);">No tasks in queue. Go to Manage Tasks to create one.</p>`; | |
| } else { | |
| let thtml = `<table><tr><th width="80">Priority</th><th>Series</th><th width="40%">Item-Level Progress</th><th>Status</th><th width="150">Actions</th></tr>`; | |
| d.tasks.forEach(t=>{ | |
| let is = t.item_stats; | |
| let ttot = is.total_items || 1; | |
| let pctDone = (is.success_items/ttot)*100; | |
| let pctProg = (is.in_progress_items/ttot)*100; | |
| let pctFail = (is.failed_items/ttot)*100; | |
| let pctPend = (is.pending_items/ttot)*100; | |
| let actions = ''; | |
| if(t.status==='paused') actions += `<button onclick="actTask('${t.id}','resume')">▶ Resume</button> `; | |
| else if(t.status==='pending'||t.status==='running') actions += `<button onclick="actTask('${t.id}','pause')">⏸ Pause</button> `; | |
| let retryBtn = (is.failed_items > 0 || is.pending_items > 0) ? `<button onclick="actTask('${t.id}','retry')" style="margin-top:5px; width:100%">🔄 Retry ${fmt(is.failed_items)} Failed</button>` : ''; | |
| thtml += `<tr> | |
| <td><span style="font-size: 16px; font-weight:700; color:var(--text)">#${t.priority}</span></td> | |
| <td> | |
| <span style="font-weight:600">${t.series_name}</span><br> | |
| <code style="color:var(--text-muted); font-size:12px;">${t.start_etp_id} → ${t.end_etp_id}</code> | |
| </td> | |
| <td> | |
| <div class="prog-stats"> | |
| <span class="item-count" style="color:var(--green)">${fmt(is.success_items)} Done</span> | |
| <span class="item-count" style="color:var(--blue)">${fmt(is.in_progress_items)} Active</span> | |
| <span class="item-count" style="color:var(--yellow)">${fmt(is.pending_items)} Pending</span> | |
| <span class="item-count" style="color:var(--red)">${fmt(is.failed_items)} Failed</span> | |
| </div> | |
| <div class="prog-bar"> | |
| <div class="prog-fill fill-green" style="width:${pctDone}%"></div> | |
| <div class="prog-fill fill-blue" style="width:${pctProg}%"></div> | |
| <div class="prog-fill fill-red" style="width:${pctFail}%"></div> | |
| <div class="prog-fill fill-yellow" style="width:${pctPend}%"></div> | |
| </div> | |
| <div style="margin-top:4px; font-size:11px; color:var(--text-muted);"> | |
| ${((is.success_items/ttot)*100).toFixed(2)}% of ${fmt(ttot)} items | |
| ${is.failed_items > 0 ? ` | <span style="color:var(--red)">${fmt(is.failed_items)} need retry</span>` : ''} | |
| </div> | |
| </td> | |
| <td><span class="badge b-${t.status}">${t.status.toUpperCase()}</span></td> | |
| <td> | |
| <div style="display:flex; gap:5px;"> | |
| ${actions} | |
| <button class="btn-danger" onclick="actTask('${t.id}','delete')" title="Delete Task">🗑</button> | |
| </div> | |
| ${retryBtn} | |
| </td> | |
| </tr>`; | |
| }); | |
| document.getElementById('tasks-table').innerHTML = thtml + `</table>`; | |
| } | |
| // Workers Table | |
| if(d.workers.length === 0){ | |
| document.getElementById('workers-table').innerHTML = `<p style="text-align:center; padding: 30px; color: var(--text-muted);">No active workers found.<br><br><span style="font-size:24px;">😴</span></p>`; | |
| } else { | |
| let whtml = `<table><tr><th>Worker ID</th><th>Platform</th><th>Status</th><th>Total Uptime</th><th>Last Heartbeat</th></tr>`; | |
| d.workers.forEach(w=>{ | |
| let activeColor = w.seconds_dead < 60 ? "var(--green)" : "var(--yellow)"; | |
| whtml += `<tr> | |
| <td><code style="color:var(--blue); background:rgba(59,130,246,0.1); padding:4px 8px; border-radius:4px;">${w.id}</code></td> | |
| <td><span style="text-transform:capitalize; font-weight:500;">${w.platform}</span></td> | |
| <td><span class="badge b-${w.status}">${w.status.toUpperCase()}</span></td> | |
| <td style="font-weight:600">${timeAgo(w.started_at)}</td> | |
| <td> | |
| <div style="display:flex; align-items:center; gap:8px;"> | |
| <div style="width:8px; height:8px; border-radius:50%; background:${activeColor};"></div> | |
| <span style="color:var(--text-muted)">${w.seconds_dead}s ago</span> | |
| </div> | |
| </td> | |
| </tr>`; | |
| }); | |
| document.getElementById('workers-table').innerHTML = whtml + `</table>`; | |
| } | |
| }).catch(e => { | |
| document.getElementById('last-sync').innerText = "Connection lost. Retrying..."; | |
| }); | |
| } | |
| function actTask(id, action) { | |
| if(action==='delete' && !confirm('WARNING: Are you sure you want to completely delete this task?')) return; | |
| fetch(`/api/tasks/${id}/action`, {method:'POST', body:JSON.stringify({action})}).then(()=>loadData()); | |
| } | |
| function toggleGlobalPause() { | |
| fetch('/api/global-pause', {method:'POST', body:JSON.stringify({paused: !globalPaused})}).then(()=>loadData()); | |
| } | |
| function createTask() { | |
| let body = { | |
| start_id: document.getElementById('t-start').value, | |
| end_id: document.getElementById('t-end').value, | |
| priority: document.getElementById('t-prio').value, | |
| chunk_size: document.getElementById('t-chunk').value | |
| }; | |
| if(!body.start_id || !body.end_id) { alert("Please enter both Start and End IDs."); return; } | |
| fetch('/api/create-task', {method:'POST', body:JSON.stringify(body)}).then(r=>r.json()).then(d=>{ | |
| if(d.ok){ alert('Task successfully added to the Queue!'); loadData(); showTab('dashboard'); } | |
| else alert('Error: ' + d.error); | |
| }); | |
| } | |
| function runSurgeon() { | |
| if(!confirm("This will force-clean any dead workers and recover stuck chunks immediately. Continue?")) return; | |
| fetch('/api/surgeon', {method:'POST'}).then(r=>r.json()).then(d=>{ | |
| alert('Surgeon finished. Recovered chunks: ' + d.recovered); loadData(); | |
| }); | |
| } | |
| // ── FAST POLLING (Every 3 seconds) ── | |
| setInterval(loadData, 3000); | |
| loadData(); | |
| </script> | |
| </body> | |
| </html> | |
| """ | |
| def root(): | |
| return HTML_CONTENT | |
| if __name__ == "__main__": | |
| uvicorn.run(app, host="0.0.0.0", port=7860) | |