GGUFGuy's picture
Duplicate from GGUFGuy/hyperdex-trainer
2a49d9a
Raw History Blame Contribute Delete
5.24 kB
{% extends "base.html" %}
{% set nav = 'queue' %}
{% block title %}Queue — NanoDex{% endblock %}
{% block body %}
<div class="sect-h" style="margin-bottom:22px">
<div>
<span class="eyebrow">Shared infrastructure</span>
<h1 style="font-size:28px;margin-top:5px">The queue, right now</h1>
</div>
{% if user %}<a class="btn primary" href="/train">+ New run</a>
{% else %}<a class="btn primary" target="_top" href="/login?next=/train">Start a run</a>{% endif %}
</div>
<div class="grid g4" style="margin-bottom:20px">
<div class="stat"><span>Training</span><b id="s-run">—</b><small>on workers</small></div>
<div class="stat"><span>Waiting</span><b id="s-q">—</b><small>in the queue</small></div>
<div class="stat"><span>Models born</span><b id="s-done">—</b><small>finished runs</small></div>
<div class="stat"><span>Tokens burned</span><b id="s-tok">—</b><small>all time</small></div>
</div>
<section style="margin-bottom:30px">
<div class="sect-h"><h2>Workers</h2></div>
<div class="grid g3" id="workers"></div>
<p class="hint" id="cache"></p>
<p class="hint" id="storage"></p>
</section>
<section style="margin-bottom:30px">
<div class="sect-h"><h2>Waiting in line</h2></div>
<div id="queued"></div>
</section>
<section>
<div class="sect-h"><h2>Recently finished</h2></div>
<div id="recent"></div>
</section>
{% endblock %}
{% block scripts %}
<script>
function tbl(rows, cols){
if(!rows.length) return '';
return `<div class="card tscroll" style="padding:6px 4px 4px"><table>
<thead><tr>${cols.map(c=>`<th>${c[0]}</th>`).join('')}</tr></thead>
<tbody>${rows.map(r=>`<tr>${cols.map(c=>`<td>${c[1](r)}</td>`).join('')}</tr>`).join('')}</tbody>
</table></div>`;
}
async function refresh(){
const q = await NX.get('/api/queue');
document.getElementById('s-run').textContent = q.stats.running;
document.getElementById('s-q').textContent = q.stats.queued;
document.getElementById('s-done').textContent = q.stats.done;
document.getElementById('s-tok').textContent = NX.fmtTokens(q.stats.tokens);
document.getElementById('workers').innerHTML = q.workers.map(w => {
if(!w.job){
return `<div class="gpu">
<div class="gpu-h"><b>Worker ${w.index}</b><span>${NX.esc(w.device)}</span></div>
<div class="muted" style="font-size:13.5px">idle — ready for the next run</div>
</div>`;
}
const j = w.job, pct = (j.progress*100).toFixed(1);
return `<div class="gpu busy">
<div class="gpu-h"><b>Worker ${w.index}</b><span>${NX.esc(w.device)}</span></div>
<div style="font-weight:620;font-size:14.5px">${NX.esc(j.name)}</div>
<div class="sub" style="font-size:12.5px">@${NX.esc(j.username)} · ${NX.esc(j.tier_label)}</div>
<div class="bar thin" style="margin:10px 0 7px"><i style="width:${pct}%"></i></div>
<div class="mono muted" style="font-size:12px">${pct}% ·
loss ${j.loss ? j.loss.toFixed(3) : '—'} ·
${Math.round(j.tok_per_s/1e3)}k tok/s</div>
</div>`;
}).join('');
const d = q.data;
const pct = Math.min(100, 100 * d.ready_tokens / (d.target||1));
document.getElementById('cache').textContent =
`fineweb-edu token cache: ${NX.fmtTokens(d.ready_tokens)} tokens ready ` +
`(${pct.toFixed(0)}% · ${d.status})`;
const st = q.store || {};
document.getElementById('storage').innerHTML = st.enabled
? `durable storage: <b>on</b> — finished models and their loss curves are
archived to <span class="mono">${NX.esc(st.bucket)}</span> and survive
restarts${st.restored ? ` (${st.restored} run${st.restored===1?'':'s'} restored on boot)` : ''}`
: `durable storage: <b style="color:var(--amber)">off</b> — this container's
history and checkpoints are lost on the next restart`;
document.getElementById('queued').innerHTML = q.queued.length
? tbl(q.queued, [
['#', r => '<b>'+(r.queue_position||'?')+'</b>'],
['Model', r => NX.esc(r.name)],
['Author', r => `<a href="/u/${encodeURIComponent(r.username)}"
style="color:var(--accent)">@${NX.esc(r.username)}</a>`],
['Size', r => NX.esc(r.tier_label)],
['Tokens', r => NX.fmtTokens(r.target_tokens)],
['Waiting', r => NX.ago(r.created_at)],
])
: `<div class="empty">Queue is empty — a worker will start your run immediately.</div>`;
document.getElementById('recent').innerHTML = q.recent.length
? tbl(q.recent, [
['Model', r => NX.esc(r.name)],
['Author', r => `<a href="/u/${encodeURIComponent(r.username)}"
style="color:var(--accent)">@${NX.esc(r.username)}</a>`],
['Size', r => NX.esc(r.tier_label)],
['Tokens', r => NX.fmtTokens(r.tokens_seen)],
['Final loss', r => r.loss ? r.loss.toFixed(4) : '—'],
['On the Hub', r => r.pushed_repo
? `<a class="mono" style="color:var(--accent)" target="_blank" rel="noopener"
href="https://huggingface.co/${NX.esc(r.pushed_repo)}">${NX.esc(r.pushed_repo)}</a>`
: '<span class="muted">not published</span>'],
])
: `<div class="empty">No finished runs yet. Someone has to be first.</div>`;
}
NX.poll(refresh, 4000);
</script>
{% endblock %}