| import os |
| import time |
| import threading |
| from fastapi import FastAPI, Request |
| from fastapi.responses import HTMLResponse, JSONResponse |
| from fastapi.middleware.cors import CORSMiddleware |
| import uvicorn |
| import traceback |
|
|
| from config import REPO_ID, BUCKET_ID, MODEL_PRESETS, DATA_SOURCES, HEARTBEAT_TIMEOUT |
| from core.state import workers, tasks |
|
|
| from api.workers import router as workers_router |
| from api.tasks import router as tasks_router |
| from api.results import router as results_router |
| from api.console import router as console_router |
|
|
| app = FastAPI(title="Nightglow AI Master", version="4.0") |
| app.add_middleware( |
| CORSMiddleware, |
| allow_origins=["*"], |
| allow_credentials=True, |
| allow_methods=["*"], |
| allow_headers=["*"], |
| ) |
|
|
| app.include_router(workers_router) |
| app.include_router(tasks_router) |
| app.include_router(results_router) |
| app.include_router(console_router) |
|
|
| def health_check_loop(): |
| while True: |
| try: |
| now = time.time() |
| for worker_id, w in list(workers.items()): |
| if now - w.get("last_heartbeat", 0) > HEARTBEAT_TIMEOUT: |
| if w.get("status") != "offline": |
| print(f"⚠️ Worker {worker_id} heartbeat timeout, marking offline") |
| w["status"] = "offline" |
| w["status_detail"] = "offline" |
| current_task = w.get("current_task") |
| if current_task and current_task in tasks: |
| if tasks[current_task].get("status") in ["assigned", "running"]: |
| tasks[current_task]["status"] = "pending" |
| tasks[current_task]["assigned_to"] = None |
| w["current_task"] = None |
| time.sleep(30) |
| except Exception as e: |
| print(f"❌ Health check error: {e}") |
| time.sleep(30) |
|
|
| threading.Thread(target=health_check_loop, daemon=True).start() |
| print("✅ Health check thread started") |
|
|
| @app.exception_handler(Exception) |
| async def global_exception_handler(request: Request, exc: Exception): |
| print(f"❌ Global error: {exc}") |
| print(traceback.format_exc()) |
| return JSONResponse( |
| status_code=500, |
| content={"error": str(exc), "detail": "Internal Server Error"} |
| ) |
|
|
| @app.get("/") |
| async def index(): |
| html = """ |
| <!DOCTYPE html> |
| <html> |
| <head><title>Nightglow AI Master</title> |
| <style> |
| *{box-sizing:border-box;} |
| body{background:#0a0a1a;color:#e0e0ff;font-family:'Segoe UI',sans-serif;padding:20px;margin:0;} |
| .container{max-width:1200px;margin:0 auto;} |
| h1{color:#a855f7;} |
| .grid{display:grid;grid-template-columns:1fr 1fr;gap:20px;margin:15px 0;} |
| .card{background:#1a1f36;border-radius:12px;border:1px solid #2a2f4a;padding:20px;} |
| .card h3{color:#8b5cf6;margin-top:0;margin-bottom:15px;} |
| .stat-row{display:flex;justify-content:space-between;padding:6px 0;border-bottom:1px solid #1a1f36;} |
| .stat-row .label{color:#6b7280;} |
| .stat-row .value{font-weight:600;} |
| .progress-bar{width:100%;height:8px;background:#1a1f36;border-radius:4px;overflow:hidden;margin:10px 0;} |
| .progress-bar .fill{height:100%;background:linear-gradient(90deg,#6c5ce7,#a855f7);border-radius:4px;} |
| .cmd-input{width:100%;padding:10px;background:#0d1225;border:1px solid #2a2f4a;border-radius:8px;color:#e0e0ff;font-family:'Courier New',monospace;margin:10px 0;} |
| .btn{padding:8px 20px;background:#8b5cf6;color:white;border:none;border-radius:8px;cursor:pointer;} |
| .btn:hover{background:#7c3aed;} |
| .btn-secondary{background:#2a2f4a;color:#9ca3af;} |
| .btn-secondary:hover{background:#3a3f5a;} |
| .btn-danger{background:#dc2626;color:white;border:none;border-radius:8px;cursor:pointer;padding:8px 20px;} |
| .btn-danger:hover{background:#b91c1c;} |
| .output{background:#0d1225;padding:10px;border-radius:8px;font-family:'Courier New',monospace;font-size:0.85em;max-height:200px;overflow-y:auto;white-space:pre-wrap;} |
| .tag{display:inline-block;padding:2px 10px;border-radius:4px;font-size:0.8em;} |
| .tag.online{background:#10b98133;color:#10b981;} |
| .tag.offline{background:#ef444433;color:#ef4444;} |
| </style> |
| </head> |
| <body> |
| <div class="container"> |
| <h1>🌙 Nightglow AI Master</h1> |
| <div style="color:#6b7280;margin-bottom:20px;">Distributed Training · Multi-Worker · Real Training</div> |
| <div class="grid"> |
| <div class="card"> |
| <h3>📊 Training Status</h3> |
| <div id="statusContent">Loading...</div> |
| </div> |
| <div class="card"> |
| <h3>🖥️ Workers</h3> |
| <div id="workerContent">Loading...</div> |
| </div> |
| </div> |
| <div class="card"> |
| <h3>📟 Console</h3> |
| <input type="text" class="cmd-input" id="cmdInput" placeholder="Enter command (help for help)" onkeypress="if(event.key==='Enter')sendCmd()"> |
| <button class="btn" onclick="sendCmd()">Execute</button> |
| <button class="btn btn-secondary" onclick="quickCmd('status')">Status</button> |
| <button class="btn btn-secondary" onclick="quickCmd('workers')">Workers</button> |
| <button class="btn btn-secondary" onclick="quickCmd('tasks')">Tasks</button> |
| <button class="btn btn-secondary" onclick="quickCmd('results')">Results</button> |
| <button class="btn btn-secondary" onclick="quickCmd('presets')">Presets</button> |
| <button class="btn btn-secondary" onclick="quickCmd('datasources')">Data Sources</button> |
| <button class="btn btn-danger" onclick="quickCmd('delete-completed')">Delete Completed</button> |
| <button class="btn btn-secondary" onclick="quickCmd('save-results')">Save Results</button> |
| <div class="output" id="cmdOutput">Type help for available commands</div> |
| </div> |
| </div> |
| <script> |
| async function sendCmd(){ |
| const input=document.getElementById('cmdInput'); |
| const output=document.getElementById('cmdOutput'); |
| const cmd=input.value.trim(); |
| if(!cmd) return; |
| output.textContent='⏳ Executing...'; |
| try{ |
| const res=await fetch('/api/console/command',{ |
| method:'POST', |
| headers:{'Content-Type':'application/json'}, |
| body:JSON.stringify({command:cmd}) |
| }); |
| const data=await res.json(); |
| output.textContent=data.output||'No output'; |
| }catch(e){ |
| output.textContent='❌ '+e.message; |
| } |
| input.value=''; |
| } |
| function quickCmd(cmd){ |
| document.getElementById('cmdInput').value=cmd; |
| sendCmd(); |
| } |
| async function loadStatus(){ |
| try{ |
| const res=await fetch('/api/tasks/status'); |
| const data=await res.json(); |
| document.getElementById('statusContent').innerHTML=` |
| <div class="stat-row"><span class="label">Status</span><span class="value">${data.training_active?'🟢 Active':'🔴 Stopped'}</span></div> |
| <div class="stat-row"><span class="label">Target</span><span class="value">${data.target||'-'}</span></div> |
| <div class="stat-row"><span class="label">Progress</span><span class="value">${data.progress?.toFixed(1)||0}%</span></div> |
| <div class="progress-bar"><div class="fill" style="width:${data.progress||0}%"></div></div> |
| <div class="stat-row"><span class="label">Tasks</span><span class="value">${data.completed_tasks||0}/${data.total_tasks||0}</span></div> |
| <div class="stat-row"><span class="label">Total completed</span><span class="value">${data.total_completed||0}</span></div> |
| `; |
| }catch(e){ |
| document.getElementById('statusContent').innerHTML='<div style="color:#ef4444;">Load failed</div>'; |
| } |
| } |
| async function loadWorkers(){ |
| try{ |
| const res=await fetch('/api/workers/list'); |
| const data=await res.json(); |
| if(!data.workers||data.workers.length===0){ |
| document.getElementById('workerContent').innerHTML='<div style="color:#6b7280;">No workers</div>'; |
| return; |
| } |
| document.getElementById('workerContent').innerHTML=data.workers.map(w=>` |
| <div style="display:flex;justify-content:space-between;padding:6px 0;border-bottom:1px solid #1a1f36;font-size:0.9em;"> |
| <span>${w.worker_id}</span> |
| <span class="tag ${w.status}">${w.status}</span> |
| <span>${w.status_detail||'idle'}</span> |
| <span>${(w.progress*100||0).toFixed(0)}%</span> |
| <span>completed: ${w.completed_tasks||0}</span> |
| ${w.current_task ? `<span class="tag">task: ${w.current_task}</span>` : ''} |
| </div> |
| `).join(''); |
| }catch(e){ |
| document.getElementById('workerContent').innerHTML='<div style="color:#ef4444;">Load failed</div>'; |
| } |
| } |
| setInterval(loadStatus, 2000); |
| setInterval(loadWorkers, 3000); |
| loadStatus(); |
| loadWorkers(); |
| </script> |
| </body> |
| </html> |
| """ |
| return HTMLResponse(content=html) |
|
|
| if __name__ == "__main__": |
| port = int(os.getenv("PORT", 7860)) |
| print("=" * 60) |
| print("🌙 Nightglow AI Master starting") |
| print("=" * 60) |
| print(f" Port: {port}") |
| print(f" Dataset repo: {REPO_ID}") |
| print(f" Bucket: {BUCKET_ID}") |
| print(f" Model presets: {len(MODEL_PRESETS)}") |
| print("=" * 60) |
| uvicorn.run(app, host="0.0.0.0", port=port) |
