Spaces:
Running
Running
| const $ = (id) => document.getElementById(id); | |
| const charts = {}; | |
| const pending = new Map(); | |
| let requestId = 0; | |
| let toastTimer = null; | |
| let lastResult = null; | |
| let lastArenaRows = []; | |
| let lastCapacityTrace = []; | |
| let lastTopologyRows = []; | |
| let lastDesignRows = []; | |
| const COLORS = { | |
| blue: "#79a7ff", | |
| blue2: "#5d8fe9", | |
| steel: "#92a2b5", | |
| green: "#69c99a", | |
| amber: "#dfb966", | |
| red: "#df7d89", | |
| gray: "#667384", | |
| grid: "rgba(140,155,175,.14)", | |
| }; | |
| const runtimePill = $("runtimePill"); | |
| const runtimeText = $("runtimeText"); | |
| const worker = new Worker("./worker.mjs", { type: "module" }); | |
| worker.addEventListener("message", (event) => { | |
| const data = event.data || {}; | |
| if (data.type === "ready") { | |
| runtimePill.classList.add("ready"); | |
| runtimeText.textContent = "Python runtime ready"; | |
| ["runBtn", "arenaBtn", "capacityBtn", "topologyBtn", "designBtn"].forEach((id) => { $(id).disabled = false; }); | |
| window.setTimeout(() => document.body.classList.add("runtime-ready"), 650); | |
| return; | |
| } | |
| if (data.type === "fatal") { | |
| runtimePill.classList.add("error"); | |
| runtimeText.textContent = "Runtime failed"; | |
| console.error(data.error); | |
| return; | |
| } | |
| if (!pending.has(data.id)) return; | |
| const { resolve, reject } = pending.get(data.id); | |
| pending.delete(data.id); | |
| data.error ? reject(new Error(data.error)) : resolve(data.result); | |
| }); | |
| function callPython(action, payload) { | |
| const id = ++requestId; | |
| return new Promise((resolve, reject) => { | |
| pending.set(id, { resolve, reject }); | |
| worker.postMessage({ id, action, payload }); | |
| }); | |
| } | |
| function num(id) { return Number($(id).value); } | |
| function boolSelect(id) { return $(id).value === "on"; } | |
| function configFromUI(overrides = {}) { | |
| return { | |
| model: $("model").value, | |
| accelerator: $("accelerator").value, | |
| prefill_accelerator: $("prefillAccelerator").value, | |
| decode_accelerator: $("decodeAccelerator").value, | |
| prefill_workers: num("prefillWorkers"), | |
| decode_workers: num("decodeWorkers"), | |
| interconnect_gbps: num("interconnect"), | |
| transfer_base_ms: num("transferBase"), | |
| topology: $("topology").value, | |
| scheduler: $("scheduler").value, | |
| quantization: $("quantization").value, | |
| prefix_cache_enabled: boolSelect("prefixCache"), | |
| shared_prefix_tokens: num("sharedPrefix"), | |
| prefix_reuse_fraction: num("prefixReuse"), | |
| arrival_process: $("arrival").value, | |
| request_rate_rps: num("rate"), | |
| duration_s: num("duration"), | |
| prompt_tokens_mean: num("promptMean"), | |
| prompt_tokens_cv: num("promptCv"), | |
| output_tokens_mean: num("outputMean"), | |
| output_tokens_cv: num("outputCv"), | |
| max_batch_size: num("maxBatch"), | |
| max_batch_tokens: num("maxBatchTokens"), | |
| chunk_size: num("chunkSize"), | |
| kv_block_tokens: num("kvBlock"), | |
| burst_multiplier: num("burstMultiplier"), | |
| burst_period_s: num("burstPeriod"), | |
| seed: num("seed"), | |
| slo_ttft_ms: num("sloTtft"), | |
| slo_e2e_ms: num("sloE2e"), | |
| slo_attainment_target: 0.99, | |
| ...overrides, | |
| }; | |
| } | |
| function fmt(value, digits = 1) { | |
| if (!Number.isFinite(value)) return "N/A"; | |
| return value.toLocaleString(undefined, { maximumFractionDigits: digits }); | |
| } | |
| function pct(value, digits = 1) { return `${fmt(value * 100, digits)}%`; } | |
| function escapeHtml(value) { | |
| return String(value).replace(/[&<>'"]/g, (c) => ({ "&": "&", "<": "<", ">": ">", "'": "'", '"': """ })[c]); | |
| } | |
| function slug(value) { | |
| return String(value).toLowerCase().replace(/[^a-z0-9]+/g, "-").replace(/^-|-$/g, ""); | |
| } | |
| function stamp() { | |
| const d = new Date(); | |
| const pad = (v) => String(v).padStart(2, "0"); | |
| return `${d.getFullYear()}${pad(d.getMonth() + 1)}${pad(d.getDate())}-${pad(d.getHours())}${pad(d.getMinutes())}${pad(d.getSeconds())}`; | |
| } | |
| function showToast(message) { | |
| const toast = $("toast"); | |
| toast.textContent = message; | |
| toast.classList.add("show"); | |
| if (toastTimer) clearTimeout(toastTimer); | |
| toastTimer = setTimeout(() => toast.classList.remove("show"), 1800); | |
| } | |
| async function copyText(text, message) { | |
| try { | |
| await navigator.clipboard.writeText(text); | |
| showToast(message); | |
| } catch { | |
| const area = document.createElement("textarea"); | |
| area.value = text; | |
| area.style.position = "fixed"; | |
| area.style.opacity = "0"; | |
| document.body.appendChild(area); | |
| area.select(); | |
| document.execCommand("copy"); | |
| area.remove(); | |
| showToast(message); | |
| } | |
| } | |
| function csvCell(value) { | |
| const text = String(value ?? ""); | |
| return /[",\n]/.test(text) ? `"${text.replaceAll('"', '""')}"` : text; | |
| } | |
| function tableText(headers, rows, separator = "\t") { | |
| return [headers, ...rows].map((row) => row.join(separator)).join("\n"); | |
| } | |
| function downloadCsv(filename, headers, rows) { | |
| const csv = [headers, ...rows].map((row) => row.map(csvCell).join(",")).join("\n"); | |
| const blob = new Blob([csv], { type: "text/csv;charset=utf-8" }); | |
| const url = URL.createObjectURL(blob); | |
| const a = document.createElement("a"); | |
| a.href = url; | |
| a.download = filename; | |
| a.click(); | |
| URL.revokeObjectURL(url); | |
| } | |
| function downloadJson(result) { | |
| const cfg = result.config || configFromUI(); | |
| const topology = cfg.topology === "disaggregated_pd" ? `pd-${cfg.prefill_workers}p-${cfg.decode_workers}d` : `colocated-${cfg.accelerator}`; | |
| const name = `inferscale-run_${slug(cfg.model)}_${slug(topology)}_${slug(cfg.scheduler)}_${stamp()}.json`; | |
| const blob = new Blob([JSON.stringify(result, null, 2)], { type: "application/json" }); | |
| const url = URL.createObjectURL(blob); | |
| const a = document.createElement("a"); | |
| a.href = url; | |
| a.download = name; | |
| a.click(); | |
| URL.revokeObjectURL(url); | |
| } | |
| Chart.defaults.color = "#929dab"; | |
| Chart.defaults.borderColor = COLORS.grid; | |
| Chart.defaults.font.family = getComputedStyle(document.body).fontFamily; | |
| Chart.defaults.animation.duration = 160; | |
| function commonChartOptions() { | |
| return { responsive: true, maintainAspectRatio: false }; | |
| } | |
| function lineLegend() { | |
| return { | |
| labels: { | |
| usePointStyle: true, | |
| pointStyle: "line", | |
| pointStyleWidth: 26, | |
| boxWidth: 26, | |
| boxHeight: 2, | |
| padding: 14, | |
| }, | |
| }; | |
| } | |
| function pointLegend() { | |
| return { | |
| labels: { | |
| usePointStyle: true, | |
| pointStyle: "circle", | |
| boxWidth: 8, | |
| boxHeight: 8, | |
| padding: 14, | |
| }, | |
| }; | |
| } | |
| function destroyChart(name) { | |
| if (charts[name]) { | |
| charts[name].destroy(); | |
| delete charts[name]; | |
| } | |
| } | |
| function chartFileName(card) { | |
| const cfg = configFromUI(); | |
| const chartName = card.dataset.chartName || "figure"; | |
| const topology = cfg.topology === "disaggregated_pd" | |
| ? `pd-${cfg.prefill_workers}p-${cfg.decode_workers}d-${cfg.prefill_accelerator}-${cfg.decode_accelerator}` | |
| : `colocated-${cfg.accelerator}`; | |
| return `inferscale_${slug(chartName)}_${slug(cfg.model)}_${slug(topology)}_${slug(cfg.scheduler)}_${stamp()}.png`; | |
| } | |
| function downloadChart(card) { | |
| const canvas = card.querySelector("canvas"); | |
| const chart = Chart.getChart(canvas); | |
| if (!chart) return; | |
| const exportCanvas = document.createElement("canvas"); | |
| exportCanvas.width = canvas.width; | |
| exportCanvas.height = canvas.height; | |
| const ctx = exportCanvas.getContext("2d"); | |
| ctx.fillStyle = "#0a0f15"; | |
| ctx.fillRect(0, 0, exportCanvas.width, exportCanvas.height); | |
| ctx.drawImage(canvas, 0, 0); | |
| exportCanvas.toBlob((blob) => { | |
| if (!blob) return; | |
| const url = URL.createObjectURL(blob); | |
| const a = document.createElement("a"); | |
| a.href = url; | |
| a.download = chartFileName(card); | |
| a.click(); | |
| URL.revokeObjectURL(url); | |
| showToast("Chart PNG downloaded"); | |
| }, "image/png"); | |
| } | |
| function renderSimulation(result) { | |
| lastResult = result; | |
| $("emptyState").classList.add("hidden"); | |
| $("resultContent").classList.remove("hidden"); | |
| $("exportBtn").disabled = false; | |
| $("copyResultBtn").disabled = false; | |
| const s = result.summary; | |
| const l = result.latency; | |
| const r = result.resource; | |
| const d = result.diagnostics || {}; | |
| $("mTtft").textContent = `${fmt(l.ttft_ms.p95)} ms`; | |
| $("mE2e").textContent = `${fmt(l.e2e_ms.p95)} ms`; | |
| $("mGoodput").textContent = `${fmt(s.goodput_rps, 2)} req/s`; | |
| $("mSlo").textContent = pct(s.slo_attainment); | |
| $("mReq").textContent = `${fmt(s.request_throughput_rps, 2)} req/s`; | |
| $("mKv").textContent = `${fmt(r.peak_kv_gb, 2)} GB`; | |
| const tag = $("runState"); | |
| tag.textContent = `${s.requests_completed}/${s.requests_generated} completed`; | |
| tag.className = `tag ${s.slo_attainment >= .99 && s.requests_unfinished === 0 ? "good" : "bad"}`; | |
| $("mBottleneck").textContent = d.label || "N/A"; | |
| $("mDiagnosis").textContent = d.explanation || "No simulator diagnosis available."; | |
| $("mRecommendation").textContent = d.recommendation ? `Next check: ${d.recommendation}` : ""; | |
| const e = d.evidence || {}; | |
| const evidence = [ | |
| `topology ${r.topology || result.config.topology || "colocated"}`, | |
| `busy ${pct(e.busy_fraction ?? s.busy_fraction)}`, | |
| `KV ${pct(e.peak_kv_utilization ?? r.peak_kv_utilization)}`, | |
| `TTFT pass ${pct(e.ttft_slo_attainment ?? s.ttft_slo_attainment)}`, | |
| `E2E pass ${pct(e.e2e_slo_attainment ?? s.e2e_slo_attainment)}`, | |
| `queue p95 ${fmt(e.queue_p95_ms ?? l.queue_ms.p95)} ms`, | |
| ]; | |
| if ((r.prefix_cache_hit_rate ?? 0) > 0) evidence.push(`cache hit ${pct(r.prefix_cache_hit_rate)}`, `prefill saved ${fmt(r.prefill_tokens_saved, 0)} tok`); | |
| if (r.topology === "disaggregated_pd") evidence.push( | |
| `prefill busy ${pct(r.prefill_busy_fraction)}`, | |
| `decode busy ${pct(r.decode_busy_fraction)}`, | |
| `link busy ${pct(r.transfer_busy_fraction)}`, | |
| `transfer p95 ${fmt(r.p95_transfer_ms, 2)} ms`, | |
| ); | |
| $("mEvidence").innerHTML = evidence.map((x) => `<span>${escapeHtml(x)}</span>`).join(""); | |
| destroyChart("latency"); | |
| charts.latency = new Chart($("latencyChart"), { | |
| type: "bar", | |
| data: { | |
| labels: ["TTFT p50", "TTFT p95", "E2E p50", "E2E p95", "Queue p95"], | |
| datasets: [{ | |
| label: "Milliseconds", | |
| data: [l.ttft_ms.p50, l.ttft_ms.p95, l.e2e_ms.p50, l.e2e_ms.p95, l.queue_ms.p95], | |
| backgroundColor: [COLORS.steel, COLORS.blue, "#8196b1", COLORS.blue2, COLORS.amber], | |
| }], | |
| }, | |
| options: { ...commonChartOptions(), plugins: { legend: { display: false } }, scales: { y: { beginAtZero: true } } }, | |
| }); | |
| const timeline = result.timeline || []; | |
| const lineDatasets = [ | |
| { label: r.topology === "disaggregated_pd" ? "Prefill queue" : "Waiting", data: timeline.map((x) => ({ x: x.time_s, y: x.waiting })), borderColor: COLORS.amber, pointRadius: 0, tension: .08, yAxisID: "y" }, | |
| { label: "Decoding", data: timeline.map((x) => ({ x: x.time_s, y: x.decoding })), borderColor: COLORS.blue, pointRadius: 0, tension: .08, yAxisID: "y" }, | |
| { label: "KV GB", data: timeline.map((x) => ({ x: x.time_s, y: x.kv_used_gb })), borderColor: COLORS.green, pointRadius: 0, tension: .08, yAxisID: "y1" }, | |
| ]; | |
| if (r.topology === "disaggregated_pd") { | |
| lineDatasets.splice(1, 0, { label: "Decode queue", data: timeline.map((x) => ({ x: x.time_s, y: x.decode_ready || 0 })), borderColor: COLORS.steel, borderDash: [4, 3], pointRadius: 0, tension: .08, yAxisID: "y" }); | |
| } | |
| destroyChart("timeline"); | |
| charts.timeline = new Chart($("timelineChart"), { | |
| type: "line", | |
| data: { datasets: lineDatasets }, | |
| options: { | |
| ...commonChartOptions(), | |
| parsing: false, | |
| interaction: { mode: "nearest", intersect: false }, | |
| plugins: { legend: lineLegend() }, | |
| scales: { | |
| x: { type: "linear", title: { display: true, text: "Virtual time (s)" } }, | |
| y: { beginAtZero: true, title: { display: true, text: "Requests" } }, | |
| y1: { beginAtZero: true, position: "right", grid: { drawOnChartArea: false }, title: { display: true, text: "KV GB" } }, | |
| }, | |
| }, | |
| }); | |
| const sample = result.requests || []; | |
| destroyChart("scatter"); | |
| charts.scatter = new Chart($("scatterChart"), { | |
| type: "scatter", | |
| data: { datasets: [{ label: "Requests", data: sample.map((x) => ({ x: x.prompt_tokens, y: x.ttft_ms })), backgroundColor: "rgba(121,167,255,.55)", pointRadius: 2.2 }] }, | |
| options: { ...commonChartOptions(), plugins: { legend: { display: false } }, scales: { x: { title: { display: true, text: "Prompt tokens" } }, y: { title: { display: true, text: "TTFT (ms)" }, beginAtZero: true } } }, | |
| }); | |
| const warnings = $("warnings"); | |
| const allWarnings = [...(result.warnings || [])]; | |
| if (result.provenance?.profile_warning) allWarnings.unshift(result.provenance.profile_warning); | |
| if (allWarnings.length) { | |
| warnings.innerHTML = allWarnings.map((w) => `<div>${escapeHtml(w)}</div>`).join(""); | |
| warnings.classList.remove("hidden"); | |
| } else { | |
| warnings.classList.add("hidden"); | |
| } | |
| } | |
| $("runBtn").addEventListener("click", async () => { | |
| const button = $("runBtn"); | |
| const state = $("runState"); | |
| button.disabled = true; | |
| button.textContent = "Running simulation..."; | |
| state.textContent = "Simulating..."; | |
| state.className = "tag neutral"; | |
| try { | |
| renderSimulation(await callPython("simulate", configFromUI())); | |
| } catch (error) { | |
| state.textContent = "Error"; | |
| state.className = "tag bad"; | |
| alert(`Simulation failed: ${error.message}`); | |
| } finally { | |
| button.textContent = "Run simulation"; | |
| button.disabled = false; | |
| } | |
| }); | |
| $("copyResultBtn").addEventListener("click", () => { if (lastResult) copyText(JSON.stringify(lastResult, null, 2), "Result JSON copied"); }); | |
| $("exportBtn").addEventListener("click", () => { if (lastResult) downloadJson(lastResult); }); | |
| function schedulerLabel(value) { | |
| return ({ | |
| static_fcfs: "Static FCFS", | |
| continuous_fcfs: "Continuous FCFS", | |
| continuous_sjf: "Continuous SJF", | |
| continuous_slo: "Continuous SLO", | |
| chunked_slo: "Chunked SLO", | |
| })[value] || value; | |
| } | |
| function arenaTableRows(rows) { | |
| return rows.map((r) => [schedulerLabel(r.scheduler), `${fmt(r.goodput_rps, 2)} req/s`, pct(r.slo_attainment), `${fmt(r.p95_ttft_ms)} ms`, `${fmt(r.p95_e2e_ms)} ms`, pct(r.peak_kv_utilization), fmt(r.unfinished, 0), r.bottleneck || "N/A"]); | |
| } | |
| function renderArena(rows) { | |
| lastArenaRows = rows; | |
| $("arenaEmpty").classList.add("hidden"); | |
| $("arenaContent").classList.remove("hidden"); | |
| $("arenaCopyBtn").disabled = false; | |
| $("arenaCsvBtn").disabled = false; | |
| $("arenaRows").innerHTML = rows.map((r, index) => `<tr><td>${escapeHtml(schedulerLabel(r.scheduler))}${index === 0 ? '<span class="best-label">Best</span>' : ""}</td><td>${fmt(r.goodput_rps, 2)} req/s</td><td>${fmt(r.goodput_per_accelerator, 2)} req/s/GPU</td><td>${pct(r.slo_attainment)}</td><td>${fmt(r.p95_ttft_ms)} ms</td><td>${fmt(r.p95_e2e_ms)} ms</td><td>${pct(r.peak_kv_utilization)}</td><td>${fmt(r.unfinished, 0)}</td><td>${escapeHtml(r.bottleneck || "N/A")}</td></tr>`).join(""); | |
| destroyChart("arena"); | |
| charts.arena = new Chart($("arenaChart"), { | |
| type: "bar", | |
| data: { labels: rows.map((r) => schedulerLabel(r.scheduler)), datasets: [{ label: "Goodput (req/s)", data: rows.map((r) => r.goodput_rps), backgroundColor: COLORS.blue }, { label: "Raw throughput (req/s)", data: rows.map((r) => r.request_throughput_rps), backgroundColor: "#53677f" }] }, | |
| options: { ...commonChartOptions(), scales: { y: { beginAtZero: true } } }, | |
| }); | |
| } | |
| $("arenaBtn").addEventListener("click", async () => { | |
| const button = $("arenaBtn"); | |
| button.disabled = true; | |
| button.textContent = "Comparing..."; | |
| try { renderArena((await callPython("compare", { config: configFromUI() })).rows); } | |
| catch (error) { alert(`Scheduler comparison failed: ${error.message}`); } | |
| finally { button.disabled = false; button.textContent = "Compare schedulers"; } | |
| }); | |
| const arenaHeaders = ["Scheduler", "Goodput", "SLO attainment", "p95 TTFT", "p95 E2E", "KV peak", "Unfinished", "Diagnosis"]; | |
| $("arenaCopyBtn").addEventListener("click", () => copyText(tableText(arenaHeaders, arenaTableRows(lastArenaRows)), "Scheduler table copied")); | |
| $("arenaCsvBtn").addEventListener("click", () => downloadCsv(`inferscale_scheduler-arena_${stamp()}.csv`, arenaHeaders, arenaTableRows(lastArenaRows))); | |
| function capacityTableRows(trace) { | |
| return trace.map((r) => [ | |
| `${fmt(r.rate_rps, 2)} req/s`, | |
| r.passed ? "PASS" : "FAIL", | |
| pct(r.slo_attainment), | |
| pct(r.slo_attainment_min), | |
| pct(r.target ?? num("targetSlo")), | |
| `${pct(r.slo_attainment_min)} - ${pct(r.slo_attainment_max)}`, | |
| `${fmt(r.goodput_rps, 2)} req/s`, | |
| `${fmt(r.p95_ttft_ms)} ms`, | |
| `${fmt(r.p95_e2e_ms)} ms`, | |
| ]); | |
| } | |
| function renderCapacity(result) { | |
| lastCapacityTrace = result.trace || []; | |
| $("plannerEmpty").classList.add("hidden"); | |
| $("plannerContent").classList.remove("hidden"); | |
| $("capacityCopyBtn").disabled = false; | |
| $("capacityCsvBtn").disabled = false; | |
| $("pCapacity").textContent = `${fmt(result.capacity_rps, 2)} req/s`; | |
| $("pRecommended").textContent = `${fmt(result.recommended_rps, 2)} req/s`; | |
| $("pHeadroom").textContent = pct(result.headroom ?? num("headroom")); | |
| $("pStatus").textContent = result.status.replaceAll("_", " "); | |
| const state = $("plannerState"); | |
| state.textContent = result.status === "ok" ? "Search complete" : result.status.replaceAll("_", " "); | |
| state.className = `tag ${result.capacity_rps > 0 ? "good" : "bad"}`; | |
| $("capacityRows").innerHTML = lastCapacityTrace.map((r) => `<tr><td>${fmt(r.rate_rps, 2)} req/s</td><td class="${r.passed ? "pass" : "fail"}">${r.passed ? "PASS" : "FAIL"}</td><td>${pct(r.slo_attainment)}</td><td>${pct(r.slo_attainment_min)}</td><td>${pct(r.target ?? num("targetSlo"))}</td><td>${pct(r.slo_attainment_min)} - ${pct(r.slo_attainment_max)}</td><td>${fmt(r.goodput_rps, 2)} req/s</td><td>${fmt(r.p95_ttft_ms)} ms</td><td>${fmt(r.p95_e2e_ms)} ms</td></tr>`).join(""); | |
| if (!lastCapacityTrace.length) return; | |
| const target = num("targetSlo"); | |
| const mean = lastCapacityTrace.map((r) => ({ x: r.rate_rps, y: r.slo_attainment })); | |
| const worst = lastCapacityTrace.map((r) => ({ x: r.rate_rps, y: r.slo_attainment_min })); | |
| const xs = lastCapacityTrace.map((r) => r.rate_rps); | |
| const minX = Math.min(...xs); | |
| const maxX = Math.max(...xs); | |
| destroyChart("capacity"); | |
| charts.capacity = new Chart($("capacityChart"), { | |
| type: "line", | |
| data: { datasets: [ | |
| { label: "Mean SLO attainment", data: mean, borderColor: COLORS.blue, backgroundColor: COLORS.blue, fill: false, tension: .06, pointRadius: 3 }, | |
| { label: "Worst repetition", data: worst, borderColor: COLORS.amber, backgroundColor: COLORS.amber, borderDash: [5, 4], fill: false, pointRadius: 2, tension: .06 }, | |
| { label: "Target", data: [{ x: minX, y: target }, { x: maxX, y: target }], borderColor: COLORS.green, backgroundColor: COLORS.green, borderDash: [6, 5], fill: false, pointRadius: 0 }, | |
| ] }, | |
| options: { ...commonChartOptions(), parsing: false, plugins: { legend: lineLegend() }, scales: { x: { type: "linear", title: { display: true, text: "Offered load (req/s)" }, ticks: { maxTicksLimit: 8 } }, y: { min: 0, max: 1, ticks: { callback: (v) => `${Math.round(v * 100)}%` } } } }, | |
| }); | |
| } | |
| $("capacityBtn").addEventListener("click", async () => { | |
| const button = $("capacityBtn"); | |
| const state = $("plannerState"); | |
| button.disabled = true; | |
| button.textContent = "Searching..."; | |
| state.textContent = "Running simulations..."; | |
| state.className = "tag neutral"; | |
| try { | |
| renderCapacity(await callPython("capacity", { | |
| config: configFromUI({ slo_attainment_target: num("targetSlo") }), | |
| min_rate: num("minRate"), | |
| max_rate: num("maxRate"), | |
| iterations: num("searchIter"), | |
| repetitions: num("repetitions"), | |
| headroom: num("headroom"), | |
| })); | |
| } catch (error) { | |
| state.textContent = "Error"; | |
| state.className = "tag bad"; | |
| alert(`Capacity search failed: ${error.message}`); | |
| } finally { | |
| button.disabled = false; | |
| button.textContent = "Find sustainable capacity"; | |
| } | |
| }); | |
| const capacityHeaders = ["Rate", "Pass", "Mean SLO", "Worst repetition", "Target", "SLO range", "Goodput", "p95 TTFT", "p95 E2E"]; | |
| $("capacityCopyBtn").addEventListener("click", () => copyText(tableText(capacityHeaders, capacityTableRows(lastCapacityTrace)), "Capacity table copied")); | |
| $("capacityCsvBtn").addEventListener("click", () => downloadCsv(`inferscale_capacity-trace_${stamp()}.csv`, capacityHeaders, capacityTableRows(lastCapacityTrace))); | |
| function scenarioLabel(value) { | |
| return ({ | |
| colocated: "Colocated", | |
| colocated_cache: "Colocated + prefix cache", | |
| disaggregated_pd: "P/D disaggregated", | |
| disaggregated_pd_cache: "P/D + prefix cache", | |
| })[value] || value; | |
| } | |
| function topologyTableRows(rows) { | |
| return rows.map((r) => [ | |
| scenarioLabel(r.scenario), | |
| fmt(r.accelerator_instances, 0), | |
| `${fmt(r.goodput_rps, 2)} req/s`, | |
| `${fmt(r.goodput_per_accelerator, 2)} req/s/GPU`, | |
| pct(r.slo_attainment), | |
| `${fmt(r.p95_ttft_ms)} ms`, | |
| `${fmt(r.p95_e2e_ms)} ms`, | |
| `${fmt(r.p95_transfer_ms, 2)} ms`, | |
| pct(r.prefix_hit_rate), | |
| `${fmt(r.prefill_tokens_saved, 0)} tok`, | |
| r.bottleneck || "N/A", | |
| ]); | |
| } | |
| function renderTopology(rows) { | |
| lastTopologyRows = rows; | |
| $("topologyEmpty").classList.add("hidden"); | |
| $("topologyContent").classList.remove("hidden"); | |
| $("topologyCopyBtn").disabled = false; | |
| $("topologyCsvBtn").disabled = false; | |
| $("topologyRows").innerHTML = rows.map((r, index) => `<tr><td>${escapeHtml(scenarioLabel(r.scenario))}${index === 0 ? '<span class="best-label">Best</span>' : ""}</td><td>${fmt(r.accelerator_instances, 0)}</td><td>${fmt(r.goodput_rps, 2)} req/s</td><td>${fmt(r.goodput_per_accelerator, 2)} req/s/GPU</td><td>${pct(r.slo_attainment)}</td><td>${fmt(r.p95_ttft_ms)} ms</td><td>${fmt(r.p95_e2e_ms)} ms</td><td>${fmt(r.p95_transfer_ms, 2)} ms</td><td>${pct(r.prefix_hit_rate)}</td><td>${fmt(r.prefill_tokens_saved, 0)} tok</td><td>${escapeHtml(r.bottleneck || "N/A")}</td></tr>`).join(""); | |
| const palette = [COLORS.blue, COLORS.green, COLORS.amber, COLORS.red]; | |
| destroyChart("topology"); | |
| charts.topology = new Chart($("topologyChart"), { | |
| type: "scatter", | |
| data: { datasets: rows.map((r, i) => ({ label: scenarioLabel(r.scenario), data: [{ x: r.p95_ttft_ms, y: r.goodput_rps }], backgroundColor: palette[i % palette.length], borderColor: palette[i % palette.length], pointRadius: 6, pointHoverRadius: 8 })) }, | |
| options: { ...commonChartOptions(), plugins: { legend: pointLegend() }, scales: { x: { title: { display: true, text: "p95 TTFT (ms)" }, beginAtZero: true }, y: { title: { display: true, text: "Goodput (req/s)" }, beginAtZero: true } } }, | |
| }); | |
| } | |
| $("topologyBtn").addEventListener("click", async () => { | |
| const button = $("topologyBtn"); | |
| button.disabled = true; | |
| button.textContent = "Comparing..."; | |
| try { renderTopology((await callPython("topology_compare", { config: configFromUI() })).rows); } | |
| catch (error) { alert(`Topology comparison failed: ${error.message}`); } | |
| finally { button.disabled = false; button.textContent = "Compare 4 scenarios"; } | |
| }); | |
| const topologyHeaders = ["Scenario", "GPU instances", "Goodput", "Goodput / GPU", "SLO attainment", "p95 TTFT", "p95 E2E", "p95 KV transfer", "Cache hit", "Prefill saved", "Diagnosis"]; | |
| $("topologyCopyBtn").addEventListener("click", () => copyText(tableText(topologyHeaders, topologyTableRows(lastTopologyRows)), "Topology table copied")); | |
| $("topologyCsvBtn").addEventListener("click", () => downloadCsv(`inferscale_topology-cache-study_${stamp()}.csv`, topologyHeaders, topologyTableRows(lastTopologyRows))); | |
| function designTableRows(rows) { | |
| return rows.map((r) => [ | |
| r.label, | |
| r.pareto ? "YES" : "NO", | |
| r.efficiency_pareto ? "YES" : "NO", | |
| r.slo_pass ? "PASS" : "FAIL", | |
| fmt(r.accelerator_instances, 0), | |
| `${fmt(r.goodput_rps, 2)} req/s`, | |
| `${fmt(r.goodput_per_accelerator, 2)} req/s/GPU`, | |
| `${fmt(r.p95_ttft_ms)} ms`, | |
| `${fmt(r.p95_e2e_ms)} ms`, | |
| pct(r.peak_kv_utilization), | |
| r.bottleneck || "N/A", | |
| ]); | |
| } | |
| function renderDesign(result) { | |
| lastDesignRows = result.rows || []; | |
| $("designEmpty").classList.add("hidden"); | |
| $("designContent").classList.remove("hidden"); | |
| $("designCopyBtn").disabled = false; | |
| $("designCsvBtn").disabled = false; | |
| $("dCandidates").textContent = fmt(result.candidate_count, 0); | |
| $("dPareto").textContent = fmt(result.pareto_count, 0); | |
| $("dEfficiencyPareto").textContent = fmt(result.efficiency_pareto_count, 0); | |
| $("designRows").innerHTML = lastDesignRows.map((r) => `<tr><td>${escapeHtml(r.label)}${r.pareto ? '<span class="pareto-label">Perf</span>' : ""}${r.efficiency_pareto ? '<span class="pareto-label">Eff</span>' : ""}</td><td class="${r.pareto ? "pass" : ""}">${r.pareto ? "YES" : "NO"}</td><td class="${r.efficiency_pareto ? "pass" : ""}">${r.efficiency_pareto ? "YES" : "NO"}</td><td class="${r.slo_pass ? "pass" : "fail"}">${r.slo_pass ? "PASS" : "FAIL"}</td><td>${fmt(r.accelerator_instances, 0)}</td><td>${fmt(r.goodput_rps, 2)} req/s</td><td>${fmt(r.goodput_per_accelerator, 2)} req/s/GPU</td><td>${fmt(r.p95_ttft_ms)} ms</td><td>${fmt(r.p95_e2e_ms)} ms</td><td>${pct(r.peak_kv_utilization)}</td><td>${escapeHtml(r.bottleneck || "N/A")}</td></tr>`).join(""); | |
| const passPoints = lastDesignRows.filter((r) => r.slo_pass).map((r) => ({ x: r.p95_ttft_ms, y: r.goodput_rps })); | |
| const failPoints = lastDesignRows.filter((r) => !r.slo_pass).map((r) => ({ x: r.p95_ttft_ms, y: r.goodput_rps })); | |
| const frontier = lastDesignRows.filter((r) => r.pareto).sort((a, b) => a.p95_ttft_ms - b.p95_ttft_ms).map((r) => ({ x: r.p95_ttft_ms, y: r.goodput_rps })); | |
| destroyChart("design"); | |
| charts.design = new Chart($("designChart"), { | |
| type: "scatter", | |
| data: { datasets: [ | |
| { label: "SLO pass", data: passPoints, backgroundColor: COLORS.green, borderColor: COLORS.green, pointRadius: 4 }, | |
| { label: "SLO fail", data: failPoints, backgroundColor: COLORS.gray, borderColor: COLORS.gray, pointRadius: 4 }, | |
| { type: "line", label: "Performance frontier", data: frontier, borderColor: COLORS.blue, backgroundColor: COLORS.blue, pointBackgroundColor: COLORS.blue, pointRadius: 5, fill: false, tension: 0 }, | |
| ] }, | |
| options: { ...commonChartOptions(), parsing: false, plugins: { legend: pointLegend() }, scales: { x: { title: { display: true, text: "p95 TTFT (ms)" }, beginAtZero: true }, y: { title: { display: true, text: "Goodput (req/s)" }, beginAtZero: true } } }, | |
| }); | |
| const efficiencyPass = lastDesignRows.filter((r) => r.slo_pass).map((r) => ({ x: r.p95_ttft_ms, y: r.goodput_per_accelerator })); | |
| const efficiencyFail = lastDesignRows.filter((r) => !r.slo_pass).map((r) => ({ x: r.p95_ttft_ms, y: r.goodput_per_accelerator })); | |
| const efficiencyFrontier = lastDesignRows.filter((r) => r.efficiency_pareto).sort((a, b) => a.p95_ttft_ms - b.p95_ttft_ms).map((r) => ({ x: r.p95_ttft_ms, y: r.goodput_per_accelerator })); | |
| destroyChart("designEfficiency"); | |
| charts.designEfficiency = new Chart($("designEfficiencyChart"), { | |
| type: "scatter", | |
| data: { datasets: [ | |
| { label: "SLO pass", data: efficiencyPass, backgroundColor: COLORS.green, borderColor: COLORS.green, pointRadius: 4 }, | |
| { label: "SLO fail", data: efficiencyFail, backgroundColor: COLORS.gray, borderColor: COLORS.gray, pointRadius: 4 }, | |
| { type: "line", label: "Efficiency frontier", data: efficiencyFrontier, borderColor: COLORS.amber, backgroundColor: COLORS.amber, pointBackgroundColor: COLORS.amber, pointRadius: 5, fill: false, tension: 0 }, | |
| ] }, | |
| options: { ...commonChartOptions(), parsing: false, plugins: { legend: pointLegend() }, scales: { x: { title: { display: true, text: "p95 TTFT (ms)" }, beginAtZero: true }, y: { title: { display: true, text: "Goodput / accelerator (req/s/GPU)" }, beginAtZero: true } } }, | |
| }); | |
| } | |
| $("designBtn").addEventListener("click", async () => { | |
| const button = $("designBtn"); | |
| button.disabled = true; | |
| button.textContent = "Exploring..."; | |
| try { renderDesign(await callPython("design_space", { config: configFromUI(), include_disaggregated: $("includePd").checked })); } | |
| catch (error) { alert(`Design-space exploration failed: ${error.message}`); } | |
| finally { button.disabled = false; button.textContent = "Explore design space"; } | |
| }); | |
| const designHeaders = ["Candidate", "Perf Pareto", "Efficiency Pareto", "SLO pass", "GPU instances", "Goodput", "Goodput / GPU", "p95 TTFT", "p95 E2E", "KV peak", "Diagnosis"]; | |
| $("designCopyBtn").addEventListener("click", () => copyText(tableText(designHeaders, designTableRows(lastDesignRows)), "Design table copied")); | |
| $("designCsvBtn").addEventListener("click", () => downloadCsv(`inferscale_design-space_${stamp()}.csv`, designHeaders, designTableRows(lastDesignRows))); | |
| function syncConditionalControls() { | |
| const pd = $("topology").value === "disaggregated_pd"; | |
| $("pdControls").classList.toggle("hidden", !pd); | |
| $("colocatedAcceleratorLabel").classList.toggle("hidden", pd); | |
| if (pd && $("scheduler").value === "static_fcfs") { | |
| $("scheduler").value = "continuous_fcfs"; | |
| showToast("Static FCFS switched to Continuous FCFS for P/D topology"); | |
| } | |
| $("prefixControls").classList.toggle("hidden", !boolSelect("prefixCache")); | |
| $("burstControls").classList.toggle("hidden", $("arrival").value !== "bursty"); | |
| } | |
| $("topology").addEventListener("change", syncConditionalControls); | |
| $("prefixCache").addEventListener("change", syncConditionalControls); | |
| $("arrival").addEventListener("change", syncConditionalControls); | |
| syncConditionalControls(); | |
| for (const tab of document.querySelectorAll(".tab")) { | |
| tab.addEventListener("click", () => { | |
| document.querySelectorAll(".tab").forEach((item) => item.classList.remove("active")); | |
| document.querySelectorAll(".tab-panel").forEach((panel) => panel.classList.remove("active")); | |
| tab.classList.add("active"); | |
| $(tab.dataset.tab).classList.add("active"); | |
| setTimeout(() => Object.values(charts).forEach((chart) => chart.resize()), 20); | |
| }); | |
| } | |
| function closeExpandedChart() { | |
| const card = document.querySelector(".chart-card.chart-expanded"); | |
| if (!card) return; | |
| card.classList.remove("chart-expanded"); | |
| const button = card.querySelector(".chart-expand"); | |
| if (button) button.textContent = "Expand"; | |
| document.body.classList.remove("chart-open"); | |
| setTimeout(() => Chart.getChart(card.querySelector("canvas"))?.resize(), 20); | |
| } | |
| for (const button of document.querySelectorAll(".chart-expand")) { | |
| button.addEventListener("click", () => { | |
| const card = button.closest(".chart-card"); | |
| const wasExpanded = card.classList.contains("chart-expanded"); | |
| closeExpandedChart(); | |
| if (!wasExpanded) { | |
| card.classList.add("chart-expanded"); | |
| button.textContent = "Close"; | |
| document.body.classList.add("chart-open"); | |
| setTimeout(() => Chart.getChart(card.querySelector("canvas"))?.resize(), 20); | |
| } | |
| }); | |
| } | |
| for (const button of document.querySelectorAll(".chart-download")) { | |
| button.addEventListener("click", () => downloadChart(button.closest(".chart-card"))); | |
| } | |
| document.addEventListener("keydown", (event) => { if (event.key === "Escape") closeExpandedChart(); }); | |