File size: 10,029 Bytes
7b8eb46 326d7ae 9f2343c 197babf 326d7ae 7b8eb46 dfaa528 326d7ae 197babf 434159e 0074054 197babf 92ea84b 620f0ec dfaa528 5af108d 197babf 92ea84b dfaa528 620f0ec dfaa528 620f0ec dfaa528 620f0ec dfaa528 620f0ec bfa9100 620f0ec bfa9100 620f0ec bfa9100 620f0ec bfa9100 620f0ec bfa9100 620f0ec bfa9100 620f0ec bfa9100 620f0ec bfa9100 620f0ec bfa9100 197babf bfa9100 620f0ec 197babf 620f0ec bfa9100 620f0ec bfa9100 620f0ec bfa9100 620f0ec bfa9100 620f0ec bfa9100 620f0ec bfa9100 620f0ec bfa9100 620f0ec bfa9100 | 1 2 3 4 5 6 7 8 9 10 11 12 13 14 15 16 17 18 19 20 21 22 23 24 25 26 27 28 29 30 31 32 33 34 35 36 37 38 39 40 41 42 43 44 45 46 47 48 49 50 51 52 53 54 55 56 57 58 59 60 61 62 63 64 65 66 67 68 69 70 71 72 73 74 75 76 77 78 79 80 81 82 83 84 85 86 87 88 89 90 91 92 93 94 95 96 97 98 99 100 101 102 103 104 105 106 107 108 109 110 111 112 113 114 115 116 117 118 119 120 121 122 123 124 125 126 127 128 129 130 131 132 133 134 135 136 137 138 139 140 141 142 143 144 145 146 147 148 149 150 151 152 153 154 155 156 157 158 159 160 161 162 163 164 165 166 167 168 169 170 171 172 173 174 175 176 177 178 179 180 181 182 183 184 185 186 187 188 189 190 191 192 193 194 195 196 197 198 199 200 201 202 203 204 205 206 207 208 | import os
import time
import threading
from fastapi import FastAPI, Request
from fastapi.responses import HTMLResponse, JSONResponse
from fastapi.middleware.cors import CORSMiddleware
import uvicorn
import traceback
from config import REPO_ID, BUCKET_ID, MODEL_PRESETS, DATA_SOURCES, HEARTBEAT_TIMEOUT
from core.state import workers, tasks
from api.workers import router as workers_router
from api.tasks import router as tasks_router
from api.results import router as results_router
from api.console import router as console_router
app = FastAPI(title="Nightglow AI Master", version="4.0")
app.add_middleware(
CORSMiddleware,
allow_origins=["*"],
allow_credentials=True,
allow_methods=["*"],
allow_headers=["*"],
)
app.include_router(workers_router)
app.include_router(tasks_router)
app.include_router(results_router)
app.include_router(console_router)
def health_check_loop():
while True:
try:
now = time.time()
for worker_id, w in list(workers.items()):
if now - w.get("last_heartbeat", 0) > HEARTBEAT_TIMEOUT:
if w.get("status") != "offline":
print(f"⚠️ Worker {worker_id} heartbeat timeout, marking offline")
w["status"] = "offline"
w["status_detail"] = "offline"
current_task = w.get("current_task")
if current_task and current_task in tasks:
if tasks[current_task].get("status") in ["assigned", "running"]:
tasks[current_task]["status"] = "pending"
tasks[current_task]["assigned_to"] = None
w["current_task"] = None
time.sleep(30)
except Exception as e:
print(f"❌ Health check error: {e}")
time.sleep(30)
threading.Thread(target=health_check_loop, daemon=True).start()
print("✅ Health check thread started")
@app.exception_handler(Exception)
async def global_exception_handler(request: Request, exc: Exception):
print(f"❌ Global error: {exc}")
print(traceback.format_exc())
return JSONResponse(
status_code=500,
content={"error": str(exc), "detail": "Internal Server Error"}
)
@app.get("/")
async def index():
html = """
<!DOCTYPE html>
<html>
<head><title>Nightglow AI Master</title>
<style>
*{box-sizing:border-box;}
body{background:#0a0a1a;color:#e0e0ff;font-family:'Segoe UI',sans-serif;padding:20px;margin:0;}
.container{max-width:1200px;margin:0 auto;}
h1{color:#a855f7;}
.grid{display:grid;grid-template-columns:1fr 1fr;gap:20px;margin:15px 0;}
.card{background:#1a1f36;border-radius:12px;border:1px solid #2a2f4a;padding:20px;}
.card h3{color:#8b5cf6;margin-top:0;margin-bottom:15px;}
.stat-row{display:flex;justify-content:space-between;padding:6px 0;border-bottom:1px solid #1a1f36;}
.stat-row .label{color:#6b7280;}
.stat-row .value{font-weight:600;}
.progress-bar{width:100%;height:8px;background:#1a1f36;border-radius:4px;overflow:hidden;margin:10px 0;}
.progress-bar .fill{height:100%;background:linear-gradient(90deg,#6c5ce7,#a855f7);border-radius:4px;}
.cmd-input{width:100%;padding:10px;background:#0d1225;border:1px solid #2a2f4a;border-radius:8px;color:#e0e0ff;font-family:'Courier New',monospace;margin:10px 0;}
.btn{padding:8px 20px;background:#8b5cf6;color:white;border:none;border-radius:8px;cursor:pointer;}
.btn:hover{background:#7c3aed;}
.btn-secondary{background:#2a2f4a;color:#9ca3af;}
.btn-secondary:hover{background:#3a3f5a;}
.btn-danger{background:#dc2626;color:white;border:none;border-radius:8px;cursor:pointer;padding:8px 20px;}
.btn-danger:hover{background:#b91c1c;}
.output{background:#0d1225;padding:10px;border-radius:8px;font-family:'Courier New',monospace;font-size:0.85em;max-height:200px;overflow-y:auto;white-space:pre-wrap;}
.tag{display:inline-block;padding:2px 10px;border-radius:4px;font-size:0.8em;}
.tag.online{background:#10b98133;color:#10b981;}
.tag.offline{background:#ef444433;color:#ef4444;}
</style>
</head>
<body>
<div class="container">
<h1>🌙 Nightglow AI Master</h1>
<div style="color:#6b7280;margin-bottom:20px;">Distributed Training · Multi-Worker · Real Training</div>
<div class="grid">
<div class="card">
<h3>📊 Training Status</h3>
<div id="statusContent">Loading...</div>
</div>
<div class="card">
<h3>🖥️ Workers</h3>
<div id="workerContent">Loading...</div>
</div>
</div>
<div class="card">
<h3>📟 Console</h3>
<input type="text" class="cmd-input" id="cmdInput" placeholder="Enter command (help for help)" onkeypress="if(event.key==='Enter')sendCmd()">
<button class="btn" onclick="sendCmd()">Execute</button>
<button class="btn btn-secondary" onclick="quickCmd('status')">Status</button>
<button class="btn btn-secondary" onclick="quickCmd('workers')">Workers</button>
<button class="btn btn-secondary" onclick="quickCmd('tasks')">Tasks</button>
<button class="btn btn-secondary" onclick="quickCmd('results')">Results</button>
<button class="btn btn-secondary" onclick="quickCmd('presets')">Presets</button>
<button class="btn btn-secondary" onclick="quickCmd('datasources')">Data Sources</button>
<button class="btn btn-danger" onclick="quickCmd('delete-completed')">Delete Completed</button>
<button class="btn btn-secondary" onclick="quickCmd('save-results')">Save Results</button>
<div class="output" id="cmdOutput">Type help for available commands</div>
</div>
</div>
<script>
async function sendCmd(){
const input=document.getElementById('cmdInput');
const output=document.getElementById('cmdOutput');
const cmd=input.value.trim();
if(!cmd) return;
output.textContent='⏳ Executing...';
try{
const res=await fetch('/api/console/command',{
method:'POST',
headers:{'Content-Type':'application/json'},
body:JSON.stringify({command:cmd})
});
const data=await res.json();
output.textContent=data.output||'No output';
}catch(e){
output.textContent='❌ '+e.message;
}
input.value='';
}
function quickCmd(cmd){
document.getElementById('cmdInput').value=cmd;
sendCmd();
}
async function loadStatus(){
try{
const res=await fetch('/api/tasks/status');
const data=await res.json();
document.getElementById('statusContent').innerHTML=`
<div class="stat-row"><span class="label">Status</span><span class="value">${data.training_active?'🟢 Active':'🔴 Stopped'}</span></div>
<div class="stat-row"><span class="label">Target</span><span class="value">${data.target||'-'}</span></div>
<div class="stat-row"><span class="label">Progress</span><span class="value">${data.progress?.toFixed(1)||0}%</span></div>
<div class="progress-bar"><div class="fill" style="width:${data.progress||0}%"></div></div>
<div class="stat-row"><span class="label">Tasks</span><span class="value">${data.completed_tasks||0}/${data.total_tasks||0}</span></div>
<div class="stat-row"><span class="label">Total completed</span><span class="value">${data.total_completed||0}</span></div>
`;
}catch(e){
document.getElementById('statusContent').innerHTML='<div style="color:#ef4444;">Load failed</div>';
}
}
async function loadWorkers(){
try{
const res=await fetch('/api/workers/list');
const data=await res.json();
if(!data.workers||data.workers.length===0){
document.getElementById('workerContent').innerHTML='<div style="color:#6b7280;">No workers</div>';
return;
}
document.getElementById('workerContent').innerHTML=data.workers.map(w=>`
<div style="display:flex;justify-content:space-between;padding:6px 0;border-bottom:1px solid #1a1f36;font-size:0.9em;">
<span>${w.worker_id}</span>
<span class="tag ${w.status}">${w.status}</span>
<span>${w.status_detail||'idle'}</span>
<span>${(w.progress*100||0).toFixed(0)}%</span>
<span>completed: ${w.completed_tasks||0}</span>
${w.current_task ? `<span class="tag">task: ${w.current_task}</span>` : ''}
</div>
`).join('');
}catch(e){
document.getElementById('workerContent').innerHTML='<div style="color:#ef4444;">Load failed</div>';
}
}
setInterval(loadStatus, 2000);
setInterval(loadWorkers, 3000);
loadStatus();
loadWorkers();
</script>
</body>
</html>
"""
return HTMLResponse(content=html)
if __name__ == "__main__":
port = int(os.getenv("PORT", 7860))
print("=" * 60)
print("🌙 Nightglow AI Master starting")
print("=" * 60)
print(f" Port: {port}")
print(f" Dataset repo: {REPO_ID}")
print(f" Bucket: {BUCKET_ID}")
print(f" Model presets: {len(MODEL_PRESETS)}")
print("=" * 60)
uvicorn.run(app, host="0.0.0.0", port=port) |