Rebuild InferScale as interactive serving simulator v0.1

.github/workflows/ci.yml

@@ -0,0 +1,26 @@
+name: CI
+
+on:
+  push:
+  pull_request:
+
+jobs:
+  test:
+    runs-on: ubuntu-latest
+    steps:
+      - uses: actions/checkout@v4
+      - uses: actions/setup-python@v5
+        with:
+          python-version: "3.12"
+      - name: Install
+        run: pip install -e '.[dev]'
+      - name: Sync browser package
+        run: python scripts/sync_web_python.py
+      - name: Ruff
+        run: ruff check src tests scripts
+      - name: Tests
+        run: pytest -q
+      - name: Release check
+        run: python scripts/release_check.py
+      - name: Browser syntax
+        run: node --check app.js && node --check worker.mjs

.gitignore

@@ -0,0 +1,10 @@
+.venv/
+__pycache__/
+*.py[cod]
+.pytest_cache/
+.ruff_cache/
+dist/
+build/
+*.egg-info/
+.DS_Store
+results/*.json

LICENSE

@@ -0,0 +1,21 @@
+MIT License
+
+Copyright (c) 2026 Archit Sharma
+
+Permission is hereby granted, free of charge, to any person obtaining a copy
+of this software and associated documentation files (the "Software"), to deal
+in the Software without restriction, including without limitation the rights
+to use, copy, modify, merge, publish, distribute, sublicense, and/or sell
+copies of the Software, and to permit persons to whom the Software is
+furnished to do so, subject to the following conditions:
+
+The above copyright notice and this permission notice shall be included in all
+copies or substantial portions of the Software.
+
+THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
+IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
+FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE
+AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
+LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM,
+OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE
+SOFTWARE.

Makefile

@@ -0,0 +1,23 @@
+.PHONY: test sync release serve demo clean
+
+test:
+	pytest -q
+
+sync:
+	python scripts/sync_web_python.py
+
+release: sync
+	pytest -q
+	python scripts/release_check.py
+	node --check app.js
+	node --check worker.mjs
+
+demo:
+	python scripts/run_simulation.py --config examples/balanced.json
+
+serve: sync
+	python -m http.server 8000
+
+clean:
+	find . -type d -name '__pycache__' -prune -exec rm -rf {} +
+	rm -rf .pytest_cache .ruff_cache build dist *.egg-info

README.md

@@ -1,13 +1,202 @@
 ---
-title: InferScale Sim
-emoji: 🌍
-colorFrom: green
-colorTo: purple
-sdk: gradio
-sdk_version: 6.24.0
-python_version: '3.12'
-app_file: app.py
+title: InferScale-Sim
+emoji: 📈
+colorFrom: indigo
+colorTo: blue
+sdk: static
+app_file: index.html
 pinned: false
+license: mit
+short_description: Interactive LLM serving simulator and SLO planner
 ---
 
-Check out the configuration reference at https://huggingface.co/docs/hub/spaces-config-reference
+# InferScale-Sim v0.1.0
+
+**Interactive LLM serving simulator and SLO-aware capacity planner — written in Python, executed entirely in the browser.**
+
+InferScale-Sim explores a practical systems question:
+
+> How do scheduling, batching, workload shape, KV-cache pressure, and hardware assumptions change tail latency and sustainable LLM-serving capacity?
+
+The public Hugging Face Space uses **no server CPU, no GPU, no API key, and no paid inference provider**. Hugging Face serves static files; Pyodide executes the same Python simulator used by the local test suite inside a Web Worker on the visitor's ordinary CPU.
+
+> [!IMPORTANT]
+> v0.1 ships with **analytical reference latency profiles**, not measured GPU calibration data. The simulator is useful for studying serving-system dynamics and relative scheduler behavior, but its absolute latency predictions must not be presented as benchmark measurements. A future calibrated profile bundle can replace the analytical backend without changing the simulator.
+
+## Why simulation?
+
+Exhaustively testing LLM serving configurations on real GPU clusters is expensive. Microsoft's **Vidur** demonstrated the motivation dramatically: its paper reports finding a LLaMA2-70B deployment configuration in about one CPU-hour where deployment-based exploration was estimated to require **42,000 GPU-hours (~$218K)**. Vidur reported <9% latency-estimation error across its evaluated range.
+
+The area has continued moving quickly. InferScale-Sim references the broader line of work including **TokenSim (2025)**, **Revati (2026)**, **LLMServingSim 2.0 (2026)**, and **Frontier (2026)**. Recent systems increasingly model disaggregation, heterogeneous hardware, runtime control logic, memory behavior, and stateful workloads.
+
+InferScale-Sim is intentionally narrower: a compact, inspectable Python implementation focused on workload dynamics, scheduling, KV-cache behavior, SLOs, and interactive configuration search.
+
+## v0.1 capabilities
+
+- deterministic **constant, Poisson, and bursty** arrival processes
+- log-normal prompt/output-length distributions
+- **static batching** baseline
+- **continuous batching** with FCFS
+- shortest-job-first scheduling
+- deadline/SLO-aware scheduling
+- **chunked prefill** interleaved with decode
+- paged KV-cache accounting and VRAM admission control
+- analytical roofline-style prefill/decode latency proxy
+- FP16 / INT8 / INT4 weight-footprint scenarios
+- TTFT, TPOT, E2E and queue latency distributions
+- throughput, output-token throughput, **goodput**, and SLO attainment
+- live queue / decode / KV-cache timeline
+- scheduler arena using an identical deterministic workload
+- binary **capacity search** with configurable safety headroom
+- JSON experiment export
+- completely client-side Hugging Face deployment via **Pyodide Web Worker**
+
+## Architecture
+
+```text
+                         Hugging Face Static Space
+                                  │
+                                  │ serves files only
+                                  ▼
+┌─────────────────────────────────────────────────────────────┐
+│                         Browser                             │
+│                                                             │
+│   UI / Chart.js              Pyodide Web Worker             │
+│          │                           │                       │
+│          │ config                    │                       │
+│          └──────────────────────────►│                       │
+│                                      ▼                       │
+│                             Python inferscale package        │
+│                                      │                       │
+│                 ┌────────────────────┼──────────────────┐    │
+│                 │                    │                  │    │
+│             workload             scheduler          KV cache │
+│                 │                    │                  │    │
+│                 └────────────────────┼──────────────────┘    │
+│                                      ▼                       │
+│                           discrete-event loop                │
+│                                      │                       │
+│                                      ▼                       │
+│                         metrics / SLO / capacity             │
+│                                      │                       │
+│          ◄───────────────────────────┘                       │
+└─────────────────────────────────────────────────────────────┘
+```
+
+## Run locally
+
+The simulator itself has **zero runtime dependencies** beyond Python 3.10+.
+
+```bash
+python -m venv .venv
+source .venv/bin/activate
+pip install -e '.[dev]'
+
+pytest -q
+ruff check src tests scripts
+python scripts/release_check.py
+```
+
+Run one simulation from Python:
+
+```bash
+python scripts/run_simulation.py
+```
+
+Serve the browser app locally:
+
+```bash
+python -m http.server 8000
+```
+
+Open `http://localhost:8000`. The first page load downloads the Pyodide runtime from its CDN; subsequent simulation runs execute locally in the Web Worker.
+
+## Deploy to Hugging Face
+
+Create a **Static Space**, then push this repository. The root metadata already contains:
+
+```yaml
+sdk: static
+app_file: index.html
+```
+
+No Hugging Face secret is required.
+
+Before every deployment:
+
+```bash
+python scripts/sync_web_python.py
+python scripts/release_check.py
+pytest -q
+```
+
+`sync_web_python.py` mirrors the canonical `src/inferscale/` package into `py/inferscale/`, which is what Pyodide imports in the deployed application. CI fails if the browser copy is stale.
+
+## Core concepts
+
+### Goodput
+
+Raw throughput alone can reward an overloaded system. InferScale therefore reports:
+
+```text
+goodput = completed requests satisfying all configured SLOs / simulated makespan
+```
+
+A configuration processing 10 req/s with only 60% SLO attainment can have lower useful capacity than one processing 7 req/s with 99% attainment.
+
+### Static vs continuous batching
+
+The static baseline admits a batch, completes that batch, and only then admits new work. Continuous batching admits new requests as slots become available between decode iterations. This exposes queueing and head-of-line effects under heterogeneous output lengths.
+
+### KV-cache model
+
+The model accounts for K/V state across layers and grouped-query attention heads. Paged configurations round live sequence lengths to configurable token blocks; the static baseline intentionally reserves each admitted request's full prompt + maximum output sequence to expose over-reservation effects.
+
+### Latency profile honesty
+
+The default backend estimates operation time from model architecture, accelerator peak FP16 throughput / memory bandwidth, quantization footprint, and conservative efficiency factors. This is a **reference analytical model**, not a substitute for real profiling. Every result carries provenance explicitly identifying this fact.
+
+The extension point is `src/inferscale/latency.py`: a later empirical profile interpolator can satisfy the same interface.
+
+## Research lineage
+
+- **Vidur: A Large-Scale Simulation Framework for LLM Inference** (2024) — simulation + predictive profiling + deployment search. https://arxiv.org/abs/2405.05465
+- **TokenSim: Enabling Hardware and Software Exploration for Large Language Model Inference Systems** (2025) — extensible scheduling/memory simulation. https://arxiv.org/abs/2503.08415
+- **Revati: Transparent GPU-Free Time-Warp Emulation for LLM Serving** (2026) — executes real serving control logic while virtualizing GPU time. https://arxiv.org/abs/2601.00397
+- **LLMServingSim 2.0** (2026) — heterogeneous/disaggregated serving, runtime interaction, memory/power modeling. https://arxiv.org/abs/2602.23036
+- **Frontier: Towards Comprehensive and Accurate LLM Inference Simulation** (2026) — modern disaggregated serving, runtime optimizations, stateful workloads, and large-scale exploration. https://arxiv.org/abs/2605.21312
+
+See [`docs/research.md`](docs/research.md) and [`docs/methodology.md`](docs/methodology.md) for project scope and limitations.
+
+## Repository
+
+```text
+.
+├── src/inferscale/          # canonical Python simulator
+├── py/inferscale/           # generated browser mirror
+├── tests/                   # deterministic simulation tests
+├── scripts/                 # local runner + release tooling
+├── docs/                    # architecture / methodology / research notes
+├── index.html               # HF Static Space entry point
+├── app.js                   # UI + charts
+├── worker.mjs               # Pyodide Web Worker bridge
+└── styles.css
+```
+
+## Roadmap
+
+**v0.1** focuses on a clean single-accelerator serving loop and explicit scheduler trade-offs.
+
+Potential v0.2 extensions, only after v0.1 validation:
+
+- empirical calibration profile import
+- prefill/decode disaggregation + KV transfer
+- prefix caching
+- multi-replica routing
+- request priorities / tenant fairness
+- trace upload (ShareGPT-style token lengths)
+- calibration-vs-held-out validation report
+
+## License
+
+MIT.

app.js

@@ -0,0 +1,215 @@
+const $ = (id) => document.getElementById(id);
+
+const runtimePill = $("runtimePill");
+const runtimeText = $("runtimeText");
+let lastResult = null;
+let charts = {};
+let requestId = 0;
+const pending = new Map();
+
+const worker = new Worker("./worker.mjs", { type: "module" });
+worker.addEventListener("message", (event) => {
+  const data = event.data || {};
+  if (data.type === "ready") {
+    runtimePill.classList.add("ready");
+    runtimeText.textContent = "Python runtime ready";
+    ["runBtn", "arenaBtn", "capacityBtn"].forEach((id) => $(id).disabled = false);
+    return;
+  }
+  if (data.type === "fatal") {
+    runtimePill.classList.add("error");
+    runtimeText.textContent = "Runtime failed";
+    console.error(data.error);
+    return;
+  }
+  if (!pending.has(data.id)) return;
+  const { resolve, reject } = pending.get(data.id);
+  pending.delete(data.id);
+  data.error ? reject(new Error(data.error)) : resolve(data.result);
+});
+
+function callPython(action, payload) {
+  const id = ++requestId;
+  return new Promise((resolve, reject) => {
+    pending.set(id, { resolve, reject });
+    worker.postMessage({ id, action, payload });
+  });
+}
+
+function num(id) { return Number($(id).value); }
+function configFromUI() {
+  return {
+    model: $("model").value,
+    accelerator: $("accelerator").value,
+    scheduler: $("scheduler").value,
+    quantization: $("quantization").value,
+    arrival_process: $("arrival").value,
+    request_rate_rps: num("rate"),
+    duration_s: num("duration"),
+    prompt_tokens_mean: num("promptMean"),
+    prompt_tokens_cv: num("promptCv"),
+    output_tokens_mean: num("outputMean"),
+    output_tokens_cv: num("outputCv"),
+    max_batch_size: num("maxBatch"),
+    max_batch_tokens: num("maxBatchTokens"),
+    chunk_size: num("chunkSize"),
+    kv_block_tokens: num("kvBlock"),
+    seed: num("seed"),
+    slo_ttft_ms: num("sloTtft"),
+    slo_e2e_ms: num("sloE2e"),
+    slo_attainment_target: num("targetSlo"),
+  };
+}
+
+function fmt(value, digits = 1) {
+  if (!Number.isFinite(value)) return "—";
+  return value.toLocaleString(undefined, { maximumFractionDigits: digits });
+}
+function pct(v, digits = 1) { return `${fmt(v * 100, digits)}%`; }
+
+function destroyChart(name) {
+  if (charts[name]) { charts[name].destroy(); delete charts[name]; }
+}
+
+const chartDefaults = {
+  color: "#8f9aab",
+  borderColor: "rgba(150,160,180,.13)",
+};
+Chart.defaults.color = chartDefaults.color;
+Chart.defaults.borderColor = chartDefaults.borderColor;
+Chart.defaults.font.family = getComputedStyle(document.body).fontFamily;
+
+function renderSimulation(result) {
+  lastResult = result;
+  $("emptyState").classList.add("hidden");
+  $("resultContent").classList.remove("hidden");
+  $("exportBtn").disabled = false;
+  const s = result.summary, l = result.latency, r = result.resource;
+  $("mTtft").textContent = `${fmt(l.ttft_ms.p95)} ms`;
+  $("mE2e").textContent = `${fmt(l.e2e_ms.p95)} ms`;
+  $("mGoodput").textContent = `${fmt(s.goodput_rps, 2)} req/s`;
+  $("mSlo").textContent = pct(s.slo_attainment);
+  $("mReq").textContent = `${fmt(s.request_throughput_rps, 2)} req/s`;
+  $("mKv").textContent = `${fmt(r.peak_kv_gb, 2)} GB`;
+  const tag = $("runState");
+  tag.textContent = `${s.requests_completed}/${s.requests_generated} completed`;
+  tag.className = `tag ${s.slo_attainment >= 0.99 && s.requests_unfinished === 0 ? "good" : "bad"}`;
+
+  destroyChart("latency");
+  charts.latency = new Chart($("latencyChart"), {
+    type: "bar",
+    data: {
+      labels: ["TTFT p50", "TTFT p95", "E2E p50", "E2E p95", "Queue p95"],
+      datasets: [{ label: "milliseconds", data: [l.ttft_ms.p50,l.ttft_ms.p95,l.e2e_ms.p50,l.e2e_ms.p95,l.queue_ms.p95], backgroundColor: ["#766ef0","#8b7cff","#3b82f6","#55c2ff","#63d9a5"], borderRadius: 5 }]
+    },
+    options: { responsive:true, maintainAspectRatio:false, plugins:{legend:{display:false}}, scales:{y:{beginAtZero:true}} }
+  });
+
+  destroyChart("timeline");
+  const timeline = result.timeline;
+  charts.timeline = new Chart($("timelineChart"), {
+    type: "line",
+    data: {
+      labels: timeline.map(x => Number(x.time_s.toFixed(2))),
+      datasets: [
+        {label:"Waiting",data:timeline.map(x=>x.waiting),borderColor:"#ffad66",pointRadius:0,tension:.15},
+        {label:"Decoding",data:timeline.map(x=>x.decoding),borderColor:"#8b7cff",pointRadius:0,tension:.15},
+        {label:"KV GB",data:timeline.map(x=>x.kv_used_gb),borderColor:"#63d9a5",pointRadius:0,tension:.15,yAxisID:"y1"}
+      ]
+    },
+    options:{responsive:true,maintainAspectRatio:false,interaction:{mode:"index",intersect:false},scales:{x:{title:{display:true,text:"virtual time (s)"}},y:{beginAtZero:true,title:{display:true,text:"requests"}},y1:{beginAtZero:true,position:"right",grid:{drawOnChartArea:false},title:{display:true,text:"KV GB"}}}}
+  });
+
+  destroyChart("scatter");
+  const sample = result.requests || [];
+  charts.scatter = new Chart($("scatterChart"), {
+    type:"scatter",
+    data:{datasets:[{label:"requests",data:sample.map(x=>({x:x.prompt_tokens,y:x.ttft_ms})),backgroundColor:"rgba(85,194,255,.55)",pointRadius:2.5}]},
+    options:{responsive:true,maintainAspectRatio:false,plugins:{legend:{display:false}},scales:{x:{title:{display:true,text:"prompt tokens"}},y:{title:{display:true,text:"TTFT (ms)"},beginAtZero:true}}}
+  });
+
+  const warnings = $("warnings");
+  const allWarnings = [...(result.warnings || [])];
+  if (result.provenance?.profile_warning) allWarnings.unshift(result.provenance.profile_warning);
+  if (allWarnings.length) {
+    warnings.innerHTML = allWarnings.map(w=>`<div>• ${escapeHtml(w)}</div>`).join("");
+    warnings.classList.remove("hidden");
+  } else warnings.classList.add("hidden");
+}
+
+function escapeHtml(value) {
+  return String(value).replace(/[&<>'"]/g, c => ({"&":"&amp;","<":"&lt;",">":"&gt;","'":"&#39;",'"':"&quot;"}[c]));
+}
+
+function setBusy(button, stateEl, busy, label) {
+  button.disabled = busy;
+  if (stateEl) {
+    stateEl.textContent = busy ? label : stateEl.textContent;
+    if (busy) stateEl.className = "tag neutral";
+  }
+}
+
+$("runBtn").addEventListener("click", async () => {
+  const btn = $("runBtn"), state = $("runState");
+  setBusy(btn,state,true,"Simulating…"); btn.textContent="Running Python simulation…";
+  try {
+    const result = await callPython("simulate", configFromUI());
+    renderSimulation(result);
+  } catch (err) {
+    state.textContent="Error"; state.className="tag bad"; alert(`Simulation failed: ${err.message}`);
+  } finally {
+    btn.textContent="Run simulation"; btn.disabled=false;
+  }
+});
+
+$("exportBtn").addEventListener("click", () => {
+  if (!lastResult) return;
+  const blob = new Blob([JSON.stringify(lastResult,null,2)], {type:"application/json"});
+  const a = document.createElement("a"); a.href=URL.createObjectURL(blob); a.download=`inferscale-${Date.now()}.json`; a.click(); URL.revokeObjectURL(a.href);
+});
+
+$("arenaBtn").addEventListener("click", async () => {
+  const btn=$("arenaBtn"); btn.disabled=true; btn.textContent="Comparing…";
+  try {
+    const result = await callPython("compare", {config:configFromUI()});
+    renderArena(result.rows);
+  } catch(err){ alert(`Scheduler comparison failed: ${err.message}`); }
+  finally { btn.disabled=false; btn.textContent="Compare schedulers"; }
+});
+
+function schedulerLabel(s) { return ({static_fcfs:"Static FCFS",continuous_fcfs:"Continuous FCFS",continuous_sjf:"Continuous SJF",continuous_slo:"Continuous SLO",chunked_slo:"Chunked SLO"})[s] || s; }
+function renderArena(rows) {
+  $("arenaEmpty").classList.add("hidden"); $("arenaContent").classList.remove("hidden");
+  $("arenaRows").innerHTML = rows.map((r,i)=>`<tr><td>${i===0?"★ ":""}${schedulerLabel(r.scheduler)}</td><td>${fmt(r.goodput_rps,2)} req/s</td><td>${pct(r.slo_attainment)}</td><td>${fmt(r.p95_ttft_ms)} ms</td><td>${fmt(r.p95_e2e_ms)} ms</td><td>${pct(r.peak_kv_utilization)}</td><td>${fmt(r.unfinished,0)}</td></tr>`).join("");
+  destroyChart("arena");
+  charts.arena = new Chart($("arenaChart"),{type:"bar",data:{labels:rows.map(r=>schedulerLabel(r.scheduler)),datasets:[{label:"Goodput (req/s)",data:rows.map(r=>r.goodput_rps),backgroundColor:"#8b7cff",borderRadius:5},{label:"Raw throughput (req/s)",data:rows.map(r=>r.request_throughput_rps),backgroundColor:"#3f7ee8",borderRadius:5}]},options:{responsive:true,maintainAspectRatio:false,scales:{y:{beginAtZero:true}}}});
+}
+
+$("capacityBtn").addEventListener("click", async () => {
+  const btn=$("capacityBtn"), state=$("plannerState");
+  btn.disabled=true; btn.textContent="Searching…"; state.textContent="Running simulations…"; state.className="tag neutral";
+  try {
+    const config=configFromUI(); config.slo_attainment_target=num("targetSlo");
+    const result=await callPython("capacity",{config,min_rate:num("minRate"),max_rate:num("maxRate"),iterations:num("searchIter"),repetitions:num("repetitions"),headroom:num("headroom")});
+    renderCapacity(result);
+  } catch(err){ state.textContent="Error";state.className="tag bad";alert(`Capacity search failed: ${err.message}`); }
+  finally { btn.disabled=false; btn.textContent="Find sustainable capacity"; }
+});
+
+function renderCapacity(result) {
+  $("plannerEmpty").classList.add("hidden"); $("plannerContent").classList.remove("hidden");
+  $("pCapacity").textContent=`${fmt(result.capacity_rps,2)} req/s`; $("pRecommended").textContent=`${fmt(result.recommended_rps,2)} req/s`; $("pHeadroom").textContent=pct(result.headroom ?? num("headroom")); $("pStatus").textContent=result.status.replaceAll("_"," ");
+  const state=$("plannerState"); state.textContent=result.status==="ok"?"Search complete":result.status.replaceAll("_"," "); state.className=`tag ${result.capacity_rps>0?"good":"bad"}`;
+  const target=num("targetSlo");
+  $("capacityRows").innerHTML=result.trace.map(r=>`<tr><td>${fmt(r.rate_rps,2)} req/s</td><td class="${r.passed?"pass":"fail"}">${r.passed?"PASS":"FAIL"}</td><td>${pct(r.slo_attainment)}</td><td>${fmt(r.goodput_rps,2)} req/s</td><td>${fmt(r.p95_ttft_ms)} ms</td><td>${fmt(r.p95_e2e_ms)} ms</td></tr>`).join("");
+  destroyChart("capacity");
+  charts.capacity=new Chart($("capacityChart"),{type:"line",data:{labels:result.trace.map(r=>r.rate_rps),datasets:[{label:"SLO attainment",data:result.trace.map(r=>r.slo_attainment),borderColor:"#8b7cff",backgroundColor:"rgba(139,124,255,.13)",fill:true,tension:.15,pointRadius:4},{label:"Target",data:result.trace.map(()=>target),borderColor:"#63d9a5",borderDash:[6,5],pointRadius:0}]},options:{responsive:true,maintainAspectRatio:false,scales:{x:{title:{display:true,text:"offered load (req/s)"}},y:{min:0,max:1,ticks:{callback:v=>`${Math.round(v*100)}%`}}}}});
+}
+
+for (const tab of document.querySelectorAll(".tab")) {
+  tab.addEventListener("click", () => {
+    document.querySelectorAll(".tab").forEach(t=>t.classList.remove("active"));
+    document.querySelectorAll(".tab-panel").forEach(p=>p.classList.remove("active"));
+    tab.classList.add("active"); $(tab.dataset.tab).classList.add("active");
+  });
+}

docs/architecture.md

@@ -0,0 +1,31 @@
+# Architecture
+
+InferScale-Sim deliberately separates **serving-system logic** from **latency estimation**.
+
+## Simulation pipeline
+
+1. `workloads.py` generates a deterministic request trace from an arrival process and token-length distributions.
+2. `simulator.py` advances virtual time through prefill and decode work.
+3. `kv_cache.py` handles admission based on model weights, KV state and accelerator memory.
+4. `latency.py` predicts the duration of each virtual prefill/decode step.
+5. `metrics.py` derives TTFT, TPOT, E2E, queueing, throughput and SLO goodput.
+6. `optimizer.py` reruns deterministic scenarios at different offered loads or scheduling policies.
+
+The discrete-event engine never sleeps for simulated compute time. If a virtual decode step costs 40 ms, the simulator advances `now += 0.040` immediately.
+
+## Browser execution
+
+The canonical source lives in `src/inferscale`. `scripts/sync_web_python.py` mirrors those modules into `py/inferscale`. A module Web Worker loads Pyodide and writes the Python modules into Pyodide's virtual filesystem before importing `inferscale.api`.
+
+The UI remains responsive because the Python simulation does not run on the browser's main thread.
+
+## Extension boundary
+
+`AnalyticalLatencyModel` is intentionally replaceable. A calibrated implementation can provide:
+
+- `prefill_seconds(token_counts)`
+- `decode_step_seconds(context_lengths)`
+- `model_weight_gb`
+- `kv_bytes_per_token()`
+
+without changing scheduling, workload or metric code.

docs/methodology.md

@@ -0,0 +1,60 @@
+# Methodology and limitations
+
+## What v0.1 simulates
+
+InferScale models request arrival, queueing, admission, prefill, autoregressive decode, dynamic batch membership, KV-cache memory and request completion. Metrics are computed from per-request virtual timestamps.
+
+## Scheduler semantics
+
+- `static_fcfs`: admits one batch and drains it before admitting new requests.
+- `continuous_fcfs`: admits FCFS work whenever decode slots become available.
+- `continuous_sjf`: prioritizes shorter prompt+output jobs at admission.
+- `continuous_slo`: prioritizes earliest E2E deadlines.
+- `chunked_slo`: combines deadline ordering with chunked prompt prefill interleaved with decode.
+
+These are pedagogically transparent approximations, not line-by-line reproductions of vLLM or SGLang schedulers.
+
+## Workload semantics
+
+Poisson arrivals use exponentially distributed inter-arrival times. Constant arrivals are evenly spaced. Bursty arrivals alternate lower and higher rate periods to expose transient queue growth.
+
+Prompt/output lengths are sampled from log-normal distributions parameterized by mean and coefficient of variation (CV), which provides positive heavy-tailed lengths without external dependencies.
+
+## Latency model
+
+The default reference model is roofline-inspired:
+
+- dense transformer FLOPs scale approximately with model parameter count and token count;
+- attention adds a context-length-dependent term;
+- decode includes a shared model-weight memory stream and context-dependent KV reads;
+- operation time is approximated from the larger of compute and memory costs plus a small launch/scheduling proxy;
+- conservative efficiency factors prevent peak-spec numbers from being treated as achieved throughput.
+
+This produces useful qualitative dynamics but is **not empirically calibrated**. Absolute milliseconds should not be cited as hardware benchmark results.
+
+## Quantization
+
+INT8 and INT4 alter model weight footprint and apply a conservative compute-overhead multiplier. v0.1 does not claim a specific kernel implementation, quantization scheme, or quality impact.
+
+## KV-cache
+
+KV bytes per token are computed as:
+
+```text
+2 × layers × KV heads × head dimension × 2 bytes
+```
+
+for K and V with FP16 KV state. Paged allocation rounds live sequence lengths to `kv_block_tokens`. Static batching reserves full prompt+requested-output capacity.
+
+## Capacity search
+
+Capacity search evaluates an offered request rate over multiple deterministic seeds and marks the rate feasible when:
+
+1. mean SLO attainment >= configured target; and
+2. no requests remain unfinished at the end of the simulated drain.
+
+A bounded binary search then estimates the highest feasible offered rate. The recommended rate applies user-configured safety headroom.
+
+## Validation status
+
+v0.1 validates software invariants and expected qualitative behavior through unit tests. It does **not** yet provide held-out GPU calibration error. Empirical profile calibration and validation are intentionally listed as v0.2 work rather than being fabricated for the initial release.

docs/research.md

@@ -0,0 +1,37 @@
+# Research context
+
+InferScale-Sim is a portfolio-scale implementation situated within a broader research line on replacing expensive deployment sweeps with modeling, simulation or emulation.
+
+## Vidur (2024)
+
+Vidur combines experimental profiling, predictive models and end-to-end inference simulation. Its paper reports <9% inference-latency estimation error across evaluated configurations and a configuration-search example where LLaMA2-70B exploration took about one CPU-hour versus an estimated 42K GPU-hours (~$218K) for deployment-based exploration.
+
+Project inspiration: explicit performance profiles, separation of prediction from event simulation, SLO-oriented configuration search.
+
+## TokenSim (2025)
+
+TokenSim emphasizes extensible exploration of scheduling and memory-management choices, with reported sub-1% error in its evaluated real-world workloads.
+
+Project inspiration: scheduler modularity and comparative policy experiments.
+
+## Revati (2026)
+
+Revati takes a different approach: instead of reimplementing every serving-system control decision, it executes real vLLM/SGLang control paths and virtualizes CUDA time. It reports <5% prediction error and 5–17× faster execution than real GPU runs in the reported evaluation.
+
+Project lesson: control-path fidelity is a real limitation of pure simulators. InferScale intentionally documents its scheduler semantics rather than claiming production-framework equivalence.
+
+## LLMServingSim 2.0 (2026)
+
+LLMServingSim 2.0 focuses on heterogeneous and disaggregated infrastructure, integrating batching, routing, placement, offloading, memory and power into the serving loop. The 2026 version reports ~0.97% average error in its validation.
+
+Project inspiration: profile-based hardware abstraction and future disaggregation support.
+
+## Frontier (2026)
+
+Frontier models modern serving structures including co-location, prefill/decode disaggregation, Attention-FFN disaggregation, runtime optimizations and stateful workloads. Its May 2026 paper reports average throughput error below 4% on a 16-H800 testbed and large improvements in end-to-end latency error over baseline simulators.
+
+Project inspiration for v0.2: prefill/decode disaggregation, KV transfer and workload-state dependencies.
+
+## Scope boundary
+
+InferScale-Sim v0.1 is not intended to compete with these research systems on fidelity or scale. Its contribution is an inspectable, dependency-light Python implementation and a zero-backend interactive interface for exploring the underlying serving dynamics.

docs/validation.md

@@ -0,0 +1,31 @@
+# v0.1 validation checklist
+
+The release check is designed to prevent deployment and provenance mistakes rather than pretend the analytical profile has empirical accuracy.
+
+## Automated
+
+- deterministic workload generation
+- constant-arrival timing
+- prefill latency monotonicity with token count
+- quantization footprint ordering
+- successful end-to-end request completion
+- queueing/tail-latency response under increased load
+- static vs continuous batching behavioral difference
+- capacity-search output sanity
+- scheduler-arena output sanity
+- Hugging Face `short_description` <= 60 characters
+- `sdk: static` metadata
+- canonical Python source == browser mirror
+- provenance remains `analytical-reference`
+- JavaScript syntax parse
+- Python compilation
+
+## Not claimed in v0.1
+
+- empirical L4/A10G/A100 latency accuracy
+- exact vLLM/SGLang scheduler equivalence
+- CUDA-kernel modeling
+- multi-GPU communication
+- P/D disaggregation
+
+Those require calibration/profiling or additional systems components and are explicitly deferred rather than approximated silently.

examples/balanced.json

@@ -0,0 +1,19 @@
+{
+  "model": "Qwen2.5-3B",
+  "accelerator": "L4",
+  "scheduler": "continuous_fcfs",
+  "quantization": "int8",
+  "arrival_process": "poisson",
+  "request_rate_rps": 4.0,
+  "duration_s": 30,
+  "prompt_tokens_mean": 512,
+  "prompt_tokens_cv": 0.5,
+  "output_tokens_mean": 64,
+  "output_tokens_cv": 0.6,
+  "max_batch_size": 16,
+  "max_batch_tokens": 8192,
+  "slo_ttft_ms": 500,
+  "slo_e2e_ms": 5000,
+  "slo_attainment_target": 0.99,
+  "seed": 7
+}

examples/bursty.json

@@ -0,0 +1,20 @@
+{
+  "model": "Qwen2.5-3B",
+  "accelerator": "L4",
+  "scheduler": "chunked_slo",
+  "quantization": "int8",
+  "arrival_process": "bursty",
+  "request_rate_rps": 3.0,
+  "duration_s": 60,
+  "prompt_tokens_mean": 1024,
+  "prompt_tokens_cv": 0.8,
+  "output_tokens_mean": 96,
+  "output_tokens_cv": 0.6,
+  "max_batch_size": 16,
+  "max_batch_tokens": 8192,
+  "chunk_size": 512,
+  "slo_ttft_ms": 750,
+  "slo_e2e_ms": 8000,
+  "slo_attainment_target": 0.95,
+  "seed": 11
+}

examples/overload.json

@@ -0,0 +1,19 @@
+{
+  "model": "Qwen2.5-3B",
+  "accelerator": "L4",
+  "scheduler": "continuous_fcfs",
+  "quantization": "int8",
+  "arrival_process": "poisson",
+  "request_rate_rps": 10.0,
+  "duration_s": 30,
+  "prompt_tokens_mean": 512,
+  "prompt_tokens_cv": 0.5,
+  "output_tokens_mean": 64,
+  "output_tokens_cv": 0.7,
+  "max_batch_size": 16,
+  "max_batch_tokens": 8192,
+  "slo_ttft_ms": 500,
+  "slo_e2e_ms": 5000,
+  "slo_attainment_target": 0.99,
+  "seed": 7
+}

index.html

@@ -0,0 +1,207 @@
+<!doctype html>
+<html lang="en">
+<head>
+  <meta charset="utf-8" />
+  <meta name="viewport" content="width=device-width, initial-scale=1" />
+  <title>InferScale-Sim</title>
+  <meta name="description" content="Interactive LLM serving simulator and SLO-aware capacity planner." />
+  <link rel="preconnect" href="https://cdn.jsdelivr.net" crossorigin>
+  <link rel="stylesheet" href="styles.css" />
+  <script src="https://cdn.jsdelivr.net/npm/chart.js@4.5.1/dist/chart.umd.min.js"></script>
+</head>
+<body>
+  <header class="topbar">
+    <div class="brand-wrap">
+      <div class="logo">IS</div>
+      <div>
+        <div class="brand">InferScale-Sim</div>
+        <div class="subtitle">LLM serving systems laboratory</div>
+      </div>
+    </div>
+    <div class="runtime-pill" id="runtimePill"><span class="dot"></span><span id="runtimeText">Loading Python runtime…</span></div>
+  </header>
+
+  <main class="shell">
+    <section class="hero">
+      <div>
+        <div class="eyebrow">v0.1.0 · browser-native Python</div>
+        <h1>Explore LLM serving dynamics without provisioning a GPU.</h1>
+        <p>Generate workloads, compare schedulers, inspect queueing and KV pressure, then search for the maximum SLO-compliant request rate. The simulator runs locally in your browser through Pyodide.</p>
+      </div>
+      <div class="hero-stat-grid">
+        <div class="hero-stat"><span>Backend</span><strong>None</strong></div>
+        <div class="hero-stat"><span>Runtime</span><strong>Python / WASM</strong></div>
+        <div class="hero-stat"><span>Data sent</span><strong>0 bytes</strong></div>
+        <div class="hero-stat"><span>Profile type</span><strong>Analytical</strong></div>
+      </div>
+    </section>
+
+    <div class="notice">
+      <strong>Reference-profile mode.</strong> v0.1 uses analytical hardware/model profiles to study systems behavior. Absolute latency values are predictions, not measured GPU benchmarks.
+    </div>
+
+    <nav class="tabs" aria-label="InferScale sections">
+      <button class="tab active" data-tab="lab">Serving Lab</button>
+      <button class="tab" data-tab="arena">Scheduler Arena</button>
+      <button class="tab" data-tab="planner">Capacity Planner</button>
+      <button class="tab" data-tab="method">Methodology</button>
+    </nav>
+
+    <section id="lab" class="tab-panel active">
+      <div class="workspace">
+        <aside class="panel controls-panel">
+          <div class="panel-title-row"><h2>Experiment</h2><span class="tag">Deterministic seed</span></div>
+
+          <div class="field-grid two">
+            <label>Model<select id="model"><option>Qwen2.5-3B</option><option>Llama-3.1-8B</option><option>Mistral-7B-v0.3</option></select></label>
+            <label>Accelerator<select id="accelerator"><option value="L4">NVIDIA L4</option><option value="A10G">NVIDIA A10G</option><option value="A100-40GB">NVIDIA A100 40GB</option></select></label>
+          </div>
+
+          <div class="field-grid two">
+            <label>Scheduler<select id="scheduler"><option value="continuous_fcfs">Continuous · FCFS</option><option value="continuous_sjf">Continuous · SJF</option><option value="continuous_slo">Continuous · SLO-aware</option><option value="chunked_slo">Chunked prefill · SLO-aware</option><option value="static_fcfs">Static batching · FCFS</option></select></label>
+            <label>Weight precision<select id="quantization"><option value="fp16">FP16</option><option value="int8" selected>INT8 scenario</option><option value="int4">INT4 scenario</option></select></label>
+          </div>
+
+          <hr />
+          <div class="section-kicker">Workload</div>
+          <div class="field-grid two">
+            <label>Arrival process<select id="arrival"><option value="poisson">Poisson</option><option value="constant">Constant</option><option value="bursty">Bursty</option></select></label>
+            <label>Request rate<input id="rate" type="number" min="0.1" step="0.1" value="4" /><span class="unit">req/s</span></label>
+            <label>Duration<input id="duration" type="number" min="2" step="1" value="30" /><span class="unit">simulated s</span></label>
+            <label>Seed<input id="seed" type="number" step="1" value="7" /></label>
+          </div>
+          <div class="field-grid two">
+            <label>Prompt mean<input id="promptMean" type="number" min="16" value="512" /><span class="unit">tokens</span></label>
+            <label>Prompt CV<input id="promptCv" type="number" min="0" max="2" step="0.05" value="0.50" /></label>
+            <label>Output mean<input id="outputMean" type="number" min="1" value="64" /><span class="unit">tokens</span></label>
+            <label>Output CV<input id="outputCv" type="number" min="0" max="2" step="0.05" value="0.60" /></label>
+          </div>
+
+          <hr />
+          <div class="section-kicker">Serving controls</div>
+          <div class="field-grid two">
+            <label>Max batch size<input id="maxBatch" type="number" min="1" max="128" value="16" /></label>
+            <label>Max batch tokens<input id="maxBatchTokens" type="number" min="128" step="128" value="8192" /></label>
+            <label>Prefill chunk<input id="chunkSize" type="number" min="64" step="64" value="512" /><span class="unit">tokens</span></label>
+            <label>KV block<input id="kvBlock" type="number" min="1" value="16" /><span class="unit">tokens</span></label>
+          </div>
+
+          <hr />
+          <div class="section-kicker">SLO</div>
+          <div class="field-grid two">
+            <label>TTFT limit<input id="sloTtft" type="number" min="1" value="500" /><span class="unit">ms</span></label>
+            <label>E2E limit<input id="sloE2e" type="number" min="100" value="5000" /><span class="unit">ms</span></label>
+          </div>
+
+          <button id="runBtn" class="primary" disabled>Run simulation</button>
+          <button id="exportBtn" class="secondary" disabled>Export last result</button>
+        </aside>
+
+        <div class="results-column">
+          <section class="panel result-panel">
+            <div class="panel-title-row"><h2>Run summary</h2><span id="runState" class="tag neutral">Waiting</span></div>
+            <div id="emptyState" class="empty-state"><div class="empty-icon">↗</div><h3>Configure a workload and run it</h3><p>The Python simulator will execute in a Web Worker and return request-level virtual timestamps.</p></div>
+            <div id="resultContent" class="hidden">
+              <div class="metric-grid">
+                <div class="metric"><span>p95 TTFT</span><strong id="mTtft">—</strong></div>
+                <div class="metric"><span>p95 E2E</span><strong id="mE2e">—</strong></div>
+                <div class="metric"><span>Goodput</span><strong id="mGoodput">—</strong></div>
+                <div class="metric"><span>SLO attainment</span><strong id="mSlo">—</strong></div>
+                <div class="metric"><span>Throughput</span><strong id="mReq">—</strong></div>
+                <div class="metric"><span>Peak KV</span><strong id="mKv">—</strong></div>
+              </div>
+              <div class="chart-grid">
+                <div class="chart-card"><div class="chart-title">Latency percentiles</div><canvas id="latencyChart"></canvas></div>
+                <div class="chart-card"><div class="chart-title">Queue & decode timeline</div><canvas id="timelineChart"></canvas></div>
+              </div>
+              <div class="chart-card full"><div class="chart-title">Request TTFT vs prompt length</div><canvas id="scatterChart"></canvas></div>
+              <div id="warnings" class="warnings hidden"></div>
+            </div>
+          </section>
+        </div>
+      </div>
+    </section>
+
+    <section id="arena" class="tab-panel">
+      <div class="panel wide-panel">
+        <div class="panel-title-row">
+          <div><div class="section-kicker">Same workload · same seed</div><h2>Scheduler Arena</h2><p class="muted">Run every v0.1 scheduler against the current Serving Lab configuration and rank by SLO attainment then goodput.</p></div>
+          <button id="arenaBtn" class="primary compact" disabled>Compare schedulers</button>
+        </div>
+        <div id="arenaEmpty" class="empty-state small"><h3>No comparison yet</h3><p>Your Serving Lab controls are reused automatically.</p></div>
+        <div id="arenaContent" class="hidden">
+          <div class="chart-card full"><canvas id="arenaChart"></canvas></div>
+          <div class="table-wrap"><table><thead><tr><th>Scheduler</th><th>Goodput</th><th>SLO attainment</th><th>p95 TTFT</th><th>p95 E2E</th><th>KV peak</th><th>Unfinished</th></tr></thead><tbody id="arenaRows"></tbody></table></div>
+        </div>
+      </div>
+    </section>
+
+    <section id="planner" class="tab-panel">
+      <div class="workspace planner-grid">
+        <aside class="panel controls-panel">
+          <div class="panel-title-row"><h2>Capacity search</h2><span class="tag">Binary search</span></div>
+          <p class="muted">Find the highest offered load that reaches the target SLO attainment and drains all generated requests.</p>
+          <label>Required SLO attainment<input id="targetSlo" type="number" min="0.5" max="1" step="0.001" value="0.99" /></label>
+          <div class="field-grid two">
+            <label>Minimum rate<input id="minRate" type="number" min="0.05" step="0.1" value="0.25" /><span class="unit">req/s</span></label>
+            <label>Maximum rate<input id="maxRate" type="number" min="0.1" step="1" value="20" /><span class="unit">req/s</span></label>
+            <label>Search iterations<input id="searchIter" type="number" min="2" max="12" value="7" /></label>
+            <label>Repetitions / rate<input id="repetitions" type="number" min="1" max="5" value="2" /></label>
+          </div>
+          <label>Safety headroom<input id="headroom" type="number" min="0" max="0.8" step="0.05" value="0.20" /><span class="unit">fraction</span></label>
+          <button id="capacityBtn" class="primary" disabled>Find sustainable capacity</button>
+        </aside>
+        <section class="panel result-panel">
+          <div class="panel-title-row"><h2>Planner result</h2><span id="plannerState" class="tag neutral">Waiting</span></div>
+          <div id="plannerEmpty" class="empty-state"><h3>No search yet</h3><p>The planner repeatedly runs the simulator at different offered loads.</p></div>
+          <div id="plannerContent" class="hidden">
+            <div class="metric-grid four">
+              <div class="metric emphasis"><span>Estimated capacity</span><strong id="pCapacity">—</strong></div>
+              <div class="metric"><span>Recommended load</span><strong id="pRecommended">—</strong></div>
+              <div class="metric"><span>Safety headroom</span><strong id="pHeadroom">—</strong></div>
+              <div class="metric"><span>Status</span><strong id="pStatus">—</strong></div>
+            </div>
+            <div class="chart-card full"><div class="chart-title">SLO attainment across searched rates</div><canvas id="capacityChart"></canvas></div>
+            <div class="table-wrap"><table><thead><tr><th>Rate</th><th>Pass</th><th>SLO attainment</th><th>Goodput</th><th>p95 TTFT</th><th>p95 E2E</th></tr></thead><tbody id="capacityRows"></tbody></table></div>
+          </div>
+        </section>
+      </div>
+    </section>
+
+    <section id="method" class="tab-panel">
+      <div class="method-grid">
+        <article class="panel prose">
+          <div class="section-kicker">What is simulated?</div>
+          <h2>A serving loop, not CUDA kernels.</h2>
+          <p>InferScale advances virtual time through request arrivals, queueing, prefill, autoregressive decode, batch membership changes, KV allocation and completion. A predicted 40 ms decode step becomes <code>simulated_time += 0.040</code>; the browser never waits 40 ms.</p>
+          <p>The latency backend is roofline-inspired and intentionally replaceable. It combines model size, attention shape, accelerator peak throughput and memory bandwidth with conservative efficiency factors. v0.1 uses it to expose systems interactions, not to claim measured L4/A100 latency.</p>
+        </article>
+        <article class="panel prose">
+          <div class="section-kicker">Why goodput?</div>
+          <h2>Throughput can reward overload.</h2>
+          <p>InferScale defines goodput as the number of completed requests that satisfy both the TTFT and end-to-end latency SLOs divided by simulated makespan. A system can increase raw throughput while simultaneously becoming less useful to latency-sensitive applications.</p>
+          <div class="formula">goodput = SLO-compliant completions / simulated time</div>
+        </article>
+        <article class="panel prose wide-method">
+          <div class="section-kicker">Research lineage</div>
+          <h2>From Vidur to modern disaggregated simulators.</h2>
+          <div class="paper-grid">
+            <div><strong>Vidur · 2024</strong><span>Predictive profiling + deployment search. Reported a LLaMA2-70B search in ~1 CPU-hour versus an estimated 42K GPU-hours (~$218K).</span></div>
+            <div><strong>TokenSim · 2025</strong><span>Extensible hardware/software exploration with scheduler and memory-management modeling.</span></div>
+            <div><strong>Revati · 2026</strong><span>GPU-free time-warp emulation that executes real serving control logic rather than reimplementing it.</span></div>
+            <div><strong>LLMServingSim 2.0 · 2026</strong><span>Heterogeneous/disaggregated infrastructure with runtime-driven batching, routing, memory and power.</span></div>
+            <div><strong>Frontier · 2026</strong><span>Modern disaggregation, runtime optimizations, stateful workloads and large-scale configuration exploration.</span></div>
+          </div>
+        </article>
+      </div>
+    </section>
+  </main>
+
+  <footer>
+    <span>InferScale-Sim v0.1.0</span>
+    <span>Simulation executes locally in your browser. No experiment data is transmitted.</span>
+  </footer>
+
+  <script type="module" src="app.js"></script>
+</body>
+</html>

py/inferscale/__init__.py

@@ -0,0 +1,15 @@
+from .api import execute, metadata
+from .models import SimulationConfig
+from .optimizer import capacity_search, compare_schedulers
+from .simulator import run_simulation
+
+__all__ = [
+    "SimulationConfig",
+    "capacity_search",
+    "compare_schedulers",
+    "execute",
+    "metadata",
+    "run_simulation",
+]
+
+__version__ = "0.1.0"

py/inferscale/api.py

@@ -0,0 +1,37 @@
+from __future__ import annotations
+
+from .optimizer import capacity_search, compare_schedulers
+from .profiles import ACCELERATORS, MODELS
+from .simulator import SCHEDULERS, run_simulation
+
+
+def metadata() -> dict:
+    return {
+        "version": "0.1.0",
+        "models": list(MODELS.keys()),
+        "accelerators": list(ACCELERATORS.keys()),
+        "schedulers": sorted(SCHEDULERS),
+        "profile_type": "analytical-reference",
+    }
+
+
+def execute(action: str, payload: dict) -> dict:
+    if action == "simulate":
+        return run_simulation(payload)
+    if action == "capacity":
+        config = payload.get("config", payload)
+        return capacity_search(
+            config,
+            min_rate=float(payload.get("min_rate", 0.25)),
+            max_rate=float(payload.get("max_rate", 32.0)),
+            iterations=int(payload.get("iterations", 8)),
+            repetitions=int(payload.get("repetitions", 2)),
+            headroom=float(payload.get("headroom", 0.20)),
+        )
+    if action == "compare":
+        config = payload.get("config", payload)
+        schedulers = payload.get("schedulers")
+        return compare_schedulers(config, schedulers)
+    if action == "metadata":
+        return metadata()
+    raise ValueError(f"Unknown action: {action}")

py/inferscale/kv_cache.py

@@ -0,0 +1,39 @@
+from __future__ import annotations
+
+import math
+
+from .latency import AnalyticalLatencyModel
+from .models import Request, SimulationConfig
+
+
+class KVCacheModel:
+    def __init__(self, latency_model: AnalyticalLatencyModel, cfg: SimulationConfig):
+        self.latency_model = latency_model
+        self.cfg = cfg
+        self.model_weight_gb = latency_model.model_weight_gb
+        total_vram = latency_model.accelerator.vram_gb
+        # Reserve non-KV space for runtime workspace and weights.
+        remaining = max(0.0, total_vram - self.model_weight_gb - 1.2)
+        self.capacity_gb = remaining * cfg.kv_memory_fraction
+
+    def _allocated_tokens(self, req: Request, include_output_reservation: bool = False) -> int:
+        live_tokens = req.prompt_tokens + req.generated_tokens
+        if self.cfg.scheduler == "static_fcfs" or include_output_reservation:
+            # Static baseline reserves the full sequence, illustrating memory
+            # fragmentation / over-reservation versus paged allocation.
+            return req.prompt_tokens + req.output_tokens
+        block = max(self.cfg.kv_block_tokens, 1)
+        return int(math.ceil(live_tokens / block) * block)
+
+    def request_gb(self, req: Request, include_output_reservation: bool = False) -> float:
+        tokens = self._allocated_tokens(req, include_output_reservation)
+        return tokens * self.latency_model.kv_bytes_per_token() / 1e9
+
+    def used_gb(self, active: list[Request], prefill_pending: list[Request] | None = None) -> float:
+        requests = list(active)
+        if prefill_pending:
+            requests.extend(prefill_pending)
+        return sum(self.request_gb(r) for r in requests)
+
+    def can_admit(self, req: Request, active: list[Request], prefill_pending: list[Request] | None = None) -> bool:
+        return self.used_gb(active, prefill_pending) + self.request_gb(req) <= self.capacity_gb + 1e-12

py/inferscale/latency.py

@@ -0,0 +1,106 @@
+from __future__ import annotations
+
+import math
+
+from .models import AcceleratorProfile, ModelProfile
+from .profiles import QUANTIZATION_BYTES, QUANTIZATION_COMPUTE_MULTIPLIER
+
+
+class AnalyticalLatencyModel:
+    """Hardware-aware analytical proxy for prefill/decode latency.
+
+    This model intentionally does *not* claim benchmark-grade accuracy. It uses
+    model architecture, accelerator peak specs, conservative efficiency factors,
+    and a roofline-style max(compute_time, memory_time) approximation. Public
+    profiles are tagged `analytical-reference` throughout the app.
+    """
+
+    def __init__(self, model: ModelProfile, accelerator: AcceleratorProfile, quantization: str = "fp16"):
+        if quantization not in QUANTIZATION_BYTES:
+            raise ValueError(f"Unsupported quantization: {quantization}")
+        self.model = model
+        self.accelerator = accelerator
+        self.quantization = quantization
+        self.weight_bytes_per_param = QUANTIZATION_BYTES[quantization]
+        self.compute_overhead = QUANTIZATION_COMPUTE_MULTIPLIER[quantization]
+
+    @property
+    def model_weight_gb(self) -> float:
+        return self.model.params_b * self.weight_bytes_per_param
+
+    def kv_bytes_per_token(self) -> float:
+        # K + V, all layers, KV heads only. KV state is assumed fp16 in v0.1.
+        return (
+            2
+            * self.model.layers
+            * self.model.kv_heads
+            * self.model.head_dim
+            * 2.0
+        )
+
+    def _compute_efficiency(self, batch_size: int, tokens: int) -> float:
+        scale = 1.0 + 0.12 * math.log2(max(batch_size, 1)) + 0.035 * math.log2(max(tokens, 1))
+        return min(0.88, self.accelerator.compute_efficiency * scale)
+
+    def _bandwidth_efficiency(self, batch_size: int) -> float:
+        scale = 1.0 + 0.06 * math.log2(max(batch_size, 1))
+        return min(0.91, self.accelerator.bandwidth_efficiency * scale)
+
+    def prefill_seconds(self, token_counts: list[int]) -> float:
+        if not token_counts:
+            return 0.0
+        batch = len(token_counts)
+        total_tokens = sum(token_counts)
+        max_seq = max(token_counts)
+
+        dense_flops = 2.0 * self.model.params_b * 1e9 * total_tokens
+        # Approximate quadratic attention component. It is small for short
+        # contexts but becomes visible at long prompts.
+        attention_flops = (
+            4.0
+            * self.model.layers
+            * self.model.hidden_size
+            * sum(t * t for t in token_counts)
+        )
+        flops = (dense_flops + attention_flops) * self.compute_overhead
+        compute = flops / (
+            self.accelerator.peak_tflops_fp16 * 1e12 * self._compute_efficiency(batch, max_seq)
+        )
+
+        weight_bytes = self.model.params_b * 1e9 * self.weight_bytes_per_param
+        activation_bytes = total_tokens * self.model.hidden_size * self.model.layers * 2.0 * 0.18
+        memory = (weight_bytes + activation_bytes) / (
+            self.accelerator.bandwidth_gbps * 1e9 * self._bandwidth_efficiency(batch)
+        )
+        # Kernel launch / scheduling proxy prevents implausibly tiny times.
+        launch = 0.0018 + 0.00008 * batch
+        return max(compute, memory * 0.28) + launch
+
+    def decode_step_seconds(self, context_lengths: list[int]) -> float:
+        if not context_lengths:
+            return 0.0
+        batch = len(context_lengths)
+        avg_context = sum(context_lengths) / batch
+
+        dense_flops = 2.0 * self.model.params_b * 1e9 * batch
+        attention_flops = (
+            4.0
+            * self.model.layers
+            * self.model.hidden_size
+            * sum(context_lengths)
+        )
+        flops = (dense_flops + attention_flops) * self.compute_overhead
+        compute = flops / (
+            self.accelerator.peak_tflops_fp16 * 1e12 * self._compute_efficiency(batch, 1)
+        )
+
+        # Decode is commonly memory-bound. We model one shared weight stream plus
+        # KV reads that scale with batch and context length.
+        weight_bytes = self.model.params_b * 1e9 * self.weight_bytes_per_param
+        kv_read_bytes = self.kv_bytes_per_token() * sum(context_lengths)
+        memory = (weight_bytes + kv_read_bytes) / (
+            self.accelerator.bandwidth_gbps * 1e9 * self._bandwidth_efficiency(batch)
+        )
+
+        launch = 0.0012 + 0.000035 * batch + 0.00000003 * avg_context
+        return max(compute, memory) + launch

py/inferscale/metrics.py

@@ -0,0 +1,79 @@
+from __future__ import annotations
+
+import math
+from statistics import mean
+
+from .models import Request, RequestMetrics, SimulationConfig
+
+
+def percentile(values: list[float], q: float) -> float:
+    if not values:
+        return 0.0
+    vals = sorted(values)
+    if len(vals) == 1:
+        return vals[0]
+    pos = (len(vals) - 1) * q
+    lo = math.floor(pos)
+    hi = math.ceil(pos)
+    if lo == hi:
+        return vals[lo]
+    frac = pos - lo
+    return vals[lo] * (1.0 - frac) + vals[hi] * frac
+
+
+def request_metrics(req: Request, cfg: SimulationConfig) -> RequestMetrics:
+    if req.first_token_time is None or req.completion_time is None:
+        raise ValueError("Request is incomplete")
+    ttft_ms = (req.first_token_time - req.arrival_time) * 1000.0
+    e2e_ms = (req.completion_time - req.arrival_time) * 1000.0
+    if req.output_tokens <= 1:
+        tpot_ms = 0.0
+    else:
+        tpot_ms = (req.completion_time - req.first_token_time) * 1000.0 / (req.output_tokens - 1)
+    prefill_start = req.first_prefill_time if req.first_prefill_time is not None else req.arrival_time
+    queue_ms = max(0.0, (prefill_start - req.arrival_time) * 1000.0)
+    met_ttft = ttft_ms <= cfg.slo_ttft_ms
+    met_e2e = e2e_ms <= cfg.slo_e2e_ms
+    return RequestMetrics(
+        request_id=req.request_id,
+        arrival_time=req.arrival_time,
+        prompt_tokens=req.prompt_tokens,
+        output_tokens=req.output_tokens,
+        ttft_ms=ttft_ms,
+        e2e_ms=e2e_ms,
+        tpot_ms=tpot_ms,
+        queue_ms=queue_ms,
+        met_ttft_slo=met_ttft,
+        met_e2e_slo=met_e2e,
+        met_all_slos=met_ttft and met_e2e,
+    )
+
+
+def summarize(completed: list[Request], cfg: SimulationConfig, makespan_s: float, busy_time_s: float) -> tuple[dict, dict]:
+    metrics = [request_metrics(r, cfg) for r in completed]
+    ttft = [m.ttft_ms for m in metrics]
+    e2e = [m.e2e_ms for m in metrics]
+    tpot = [m.tpot_ms for m in metrics]
+    queue = [m.queue_ms for m in metrics]
+    total_output = sum(r.output_tokens for r in completed)
+    met = sum(m.met_all_slos for m in metrics)
+    duration = max(makespan_s, 1e-9)
+
+    summary = {
+        "requests_completed": len(completed),
+        "request_throughput_rps": len(completed) / duration,
+        "output_throughput_tps": total_output / duration,
+        "goodput_rps": met / duration,
+        "slo_attainment": met / len(metrics) if metrics else 0.0,
+        "simulated_makespan_s": makespan_s,
+        "busy_fraction": min(1.0, busy_time_s / duration),
+        "mean_prompt_tokens": mean([r.prompt_tokens for r in completed]) if completed else 0.0,
+        "mean_output_tokens": mean([r.output_tokens for r in completed]) if completed else 0.0,
+    }
+    latency = {
+        "ttft_ms": {"p50": percentile(ttft, 0.50), "p95": percentile(ttft, 0.95), "p99": percentile(ttft, 0.99)},
+        "e2e_ms": {"p50": percentile(e2e, 0.50), "p95": percentile(e2e, 0.95), "p99": percentile(e2e, 0.99)},
+        "tpot_ms": {"p50": percentile(tpot, 0.50), "p95": percentile(tpot, 0.95), "p99": percentile(tpot, 0.99)},
+        "queue_ms": {"p50": percentile(queue, 0.50), "p95": percentile(queue, 0.95), "p99": percentile(queue, 0.99)},
+    }
+    return summary, latency

py/inferscale/models.py

@@ -0,0 +1,130 @@
+from __future__ import annotations
+
+from dataclasses import asdict, dataclass, field
+from typing import Any
+
+
+@dataclass
+class ModelProfile:
+    name: str
+    params_b: float
+    layers: int
+    hidden_size: int
+    attention_heads: int
+    kv_heads: int
+    default_dtype_bytes: float = 2.0
+    source: str = "analytical-reference"
+
+    @property
+    def head_dim(self) -> int:
+        return self.hidden_size // self.attention_heads
+
+
+@dataclass
+class AcceleratorProfile:
+    name: str
+    vram_gb: float
+    peak_tflops_fp16: float
+    bandwidth_gbps: float
+    compute_efficiency: float
+    bandwidth_efficiency: float
+    source: str = "vendor-spec-reference"
+
+
+@dataclass
+class SimulationConfig:
+    model: str = "Llama-3.1-8B"
+    accelerator: str = "L4"
+    scheduler: str = "continuous_fcfs"
+    arrival_process: str = "poisson"
+    request_rate_rps: float = 4.0
+    duration_s: float = 60.0
+    prompt_tokens_mean: int = 512
+    prompt_tokens_cv: float = 0.35
+    output_tokens_mean: int = 128
+    output_tokens_cv: float = 0.35
+    max_batch_size: int = 16
+    max_batch_tokens: int = 8192
+    chunk_size: int = 512
+    kv_block_tokens: int = 16
+    kv_memory_fraction: float = 0.85
+    quantization: str = "fp16"
+    seed: int = 7
+    slo_ttft_ms: float = 500.0
+    slo_e2e_ms: float = 8000.0
+    slo_attainment_target: float = 0.99
+    burst_multiplier: float = 3.0
+    burst_period_s: float = 10.0
+    timeline_points: int = 300
+
+    @classmethod
+    def from_dict(cls, data: dict[str, Any]) -> "SimulationConfig":
+        allowed = cls.__dataclass_fields__.keys()
+        return cls(**{k: data[k] for k in allowed if k in data})
+
+    def to_dict(self) -> dict[str, Any]:
+        return asdict(self)
+
+
+@dataclass
+class Request:
+    request_id: int
+    arrival_time: float
+    prompt_tokens: int
+    output_tokens: int
+    deadline_time: float
+    remaining_prefill: int
+    generated_tokens: int = 0
+    first_prefill_time: float | None = None
+    first_token_time: float | None = None
+    completion_time: float | None = None
+    priority: int = 0
+
+    @property
+    def context_tokens(self) -> int:
+        return self.prompt_tokens + self.generated_tokens
+
+    @property
+    def complete(self) -> bool:
+        return self.generated_tokens >= self.output_tokens
+
+
+@dataclass
+class RequestMetrics:
+    request_id: int
+    arrival_time: float
+    prompt_tokens: int
+    output_tokens: int
+    ttft_ms: float
+    e2e_ms: float
+    tpot_ms: float
+    queue_ms: float
+    met_ttft_slo: bool
+    met_e2e_slo: bool
+    met_all_slos: bool
+
+
+@dataclass
+class TimelinePoint:
+    time_s: float
+    waiting: int
+    prefill_pending: int
+    decoding: int
+    completed: int
+    kv_used_gb: float
+    kv_capacity_gb: float
+
+
+@dataclass
+class SimulationResult:
+    config: dict[str, Any]
+    provenance: dict[str, Any]
+    summary: dict[str, Any]
+    latency: dict[str, Any]
+    resource: dict[str, Any]
+    requests: list[dict[str, Any]] = field(default_factory=list)
+    timeline: list[dict[str, Any]] = field(default_factory=list)
+    warnings: list[str] = field(default_factory=list)
+
+    def to_dict(self) -> dict[str, Any]:
+        return asdict(self)

py/inferscale/optimizer.py

@@ -0,0 +1,106 @@
+from __future__ import annotations
+
+from copy import deepcopy
+from statistics import mean
+
+from .models import SimulationConfig
+from .simulator import run_simulation
+
+
+def evaluate_rate(base: SimulationConfig, rate: float, repetitions: int = 3) -> dict:
+    runs = []
+    for rep in range(repetitions):
+        cfg = deepcopy(base)
+        cfg.request_rate_rps = rate
+        cfg.seed = base.seed + rep * 101
+        runs.append(run_simulation(cfg.to_dict()))
+    attainment = mean(r["summary"]["slo_attainment"] for r in runs)
+    goodput = mean(r["summary"]["goodput_rps"] for r in runs)
+    p95_ttft = mean(r["latency"]["ttft_ms"]["p95"] for r in runs)
+    p95_e2e = mean(r["latency"]["e2e_ms"]["p95"] for r in runs)
+    unfinished = mean(r["summary"]["requests_unfinished"] for r in runs)
+    passed = attainment >= base.slo_attainment_target and unfinished <= 0.0
+    return {
+        "rate_rps": rate,
+        "passed": passed,
+        "slo_attainment": attainment,
+        "goodput_rps": goodput,
+        "p95_ttft_ms": p95_ttft,
+        "p95_e2e_ms": p95_e2e,
+        "mean_unfinished": unfinished,
+    }
+
+
+def capacity_search(config: dict, min_rate: float = 0.25, max_rate: float = 32.0, iterations: int = 8, repetitions: int = 2, headroom: float = 0.20) -> dict:
+    base = SimulationConfig.from_dict(config)
+    low = max(0.01, min_rate)
+    high = max(low * 1.01, max_rate)
+    trace: list[dict] = []
+
+    low_eval = evaluate_rate(base, low, repetitions)
+    trace.append(low_eval)
+    if not low_eval["passed"]:
+        return {
+            "status": "no_feasible_rate",
+            "capacity_rps": 0.0,
+            "recommended_rps": 0.0,
+            "trace": trace,
+        }
+
+    high_eval = evaluate_rate(base, high, repetitions)
+    trace.append(high_eval)
+    if high_eval["passed"]:
+        capacity = high
+        return {
+            "status": "upper_bound_still_feasible",
+            "capacity_rps": capacity,
+            "recommended_rps": capacity * (1.0 - headroom),
+            "trace": sorted(trace, key=lambda x: x["rate_rps"]),
+        }
+
+    best = low
+    for _ in range(max(iterations, 1)):
+        mid = (low + high) / 2.0
+        result = evaluate_rate(base, mid, repetitions)
+        trace.append(result)
+        if result["passed"]:
+            best = mid
+            low = mid
+        else:
+            high = mid
+
+    return {
+        "status": "ok",
+        "capacity_rps": best,
+        "recommended_rps": best * (1.0 - headroom),
+        "headroom": headroom,
+        "trace": sorted(trace, key=lambda x: x["rate_rps"]),
+    }
+
+
+def compare_schedulers(config: dict, schedulers: list[str] | None = None) -> dict:
+    schedulers = schedulers or [
+        "static_fcfs",
+        "continuous_fcfs",
+        "continuous_sjf",
+        "continuous_slo",
+        "chunked_slo",
+    ]
+    base = SimulationConfig.from_dict(config)
+    rows = []
+    for scheduler in schedulers:
+        cfg = deepcopy(base)
+        cfg.scheduler = scheduler
+        result = run_simulation(cfg.to_dict())
+        rows.append({
+            "scheduler": scheduler,
+            "request_throughput_rps": result["summary"]["request_throughput_rps"],
+            "goodput_rps": result["summary"]["goodput_rps"],
+            "slo_attainment": result["summary"]["slo_attainment"],
+            "p95_ttft_ms": result["latency"]["ttft_ms"]["p95"],
+            "p95_e2e_ms": result["latency"]["e2e_ms"]["p95"],
+            "peak_kv_utilization": result["resource"]["peak_kv_utilization"],
+            "unfinished": result["summary"]["requests_unfinished"],
+        })
+    rows.sort(key=lambda r: (r["slo_attainment"], r["goodput_rps"]), reverse=True)
+    return {"rows": rows}

py/inferscale/profiles.py

@@ -0,0 +1,88 @@
+from __future__ import annotations
+
+from .models import AcceleratorProfile, ModelProfile
+
+
+MODELS: dict[str, ModelProfile] = {
+    "Llama-3.1-8B": ModelProfile(
+        name="Llama-3.1-8B",
+        params_b=8.03,
+        layers=32,
+        hidden_size=4096,
+        attention_heads=32,
+        kv_heads=8,
+    ),
+    "Mistral-7B-v0.3": ModelProfile(
+        name="Mistral-7B-v0.3",
+        params_b=7.25,
+        layers=32,
+        hidden_size=4096,
+        attention_heads=32,
+        kv_heads=8,
+    ),
+    "Qwen2.5-3B": ModelProfile(
+        name="Qwen2.5-3B",
+        params_b=3.09,
+        layers=36,
+        hidden_size=2048,
+        attention_heads=16,
+        kv_heads=2,
+    ),
+}
+
+
+ACCELERATORS: dict[str, AcceleratorProfile] = {
+    "L4": AcceleratorProfile(
+        name="NVIDIA L4",
+        vram_gb=24.0,
+        peak_tflops_fp16=121.0,
+        bandwidth_gbps=300.0,
+        compute_efficiency=0.40,
+        bandwidth_efficiency=0.72,
+    ),
+    "A10G": AcceleratorProfile(
+        name="NVIDIA A10G",
+        vram_gb=24.0,
+        peak_tflops_fp16=125.0,
+        bandwidth_gbps=600.0,
+        compute_efficiency=0.40,
+        bandwidth_efficiency=0.70,
+    ),
+    "A100-40GB": AcceleratorProfile(
+        name="NVIDIA A100 40GB",
+        vram_gb=40.0,
+        peak_tflops_fp16=312.0,
+        bandwidth_gbps=1555.0,
+        compute_efficiency=0.47,
+        bandwidth_efficiency=0.76,
+    ),
+}
+
+
+QUANTIZATION_BYTES = {
+    "fp16": 2.0,
+    "int8": 1.0,
+    "int4": 0.5,
+}
+
+# Compute dequantization / packing overheads are deliberately conservative analytical
+# modifiers, not empirical benchmark claims.
+QUANTIZATION_COMPUTE_MULTIPLIER = {
+    "fp16": 1.00,
+    "int8": 1.07,
+    "int4": 1.16,
+}
+
+
+def get_model(name: str) -> ModelProfile:
+    try:
+        return MODELS[name]
+    except KeyError as exc:
+        raise ValueError(f"Unknown model profile: {name}") from exc
+
+
+def get_accelerator(name: str) -> AcceleratorProfile:
+    try:
+        return ACCELERATORS[name]
+    except KeyError as exc:
+        raise ValueError(f"Unknown accelerator profile: {name}") from exc

py/inferscale/simulator.py

@@ -0,0 +1,321 @@
+from __future__ import annotations
+
+from dataclasses import asdict
+
+from .kv_cache import KVCacheModel
+from .latency import AnalyticalLatencyModel
+from .metrics import summarize
+from .models import Request, SimulationConfig, SimulationResult, TimelinePoint
+from .profiles import get_accelerator, get_model
+from .workloads import generate_workload
+
+
+SCHEDULERS = {
+    "static_fcfs",
+    "continuous_fcfs",
+    "continuous_sjf",
+    "continuous_slo",
+    "chunked_slo",
+}
+
+
+class Simulator:
+    def __init__(self, cfg: SimulationConfig):
+        if cfg.scheduler not in SCHEDULERS:
+            raise ValueError(f"Unsupported scheduler: {cfg.scheduler}")
+        self.cfg = cfg
+        self.model = get_model(cfg.model)
+        self.accelerator = get_accelerator(cfg.accelerator)
+        self.latency = AnalyticalLatencyModel(self.model, self.accelerator, cfg.quantization)
+        self.kv = KVCacheModel(self.latency, cfg)
+        self.requests = generate_workload(cfg)
+        self.pending_idx = 0
+        self.waiting: list[Request] = []
+        self.prefill_pending: list[Request] = []
+        self.active: list[Request] = []
+        self.completed: list[Request] = []
+        self.now = 0.0
+        self.busy_time = 0.0
+        self.peak_kv_gb = 0.0
+        self.timeline: list[TimelinePoint] = []
+        self.warnings: list[str] = []
+
+    def _admit_arrivals(self) -> None:
+        while self.pending_idx < len(self.requests) and self.requests[self.pending_idx].arrival_time <= self.now + 1e-12:
+            self.waiting.append(self.requests[self.pending_idx])
+            self.pending_idx += 1
+
+    def _next_arrival(self) -> float | None:
+        if self.pending_idx >= len(self.requests):
+            return None
+        return self.requests[self.pending_idx].arrival_time
+
+    def _waiting_sorted(self) -> list[Request]:
+        if self.cfg.scheduler == "continuous_sjf":
+            return sorted(self.waiting, key=lambda r: (r.prompt_tokens + r.output_tokens, r.arrival_time))
+        if self.cfg.scheduler in {"continuous_slo", "chunked_slo"}:
+            # Least-slack-first proxy: deadline minus an analytical estimate of
+            # remaining standalone service. Unlike plain EDF, this distinguishes
+            # requests with the same relative SLO but heterogeneous token lengths.
+            def slack(req: Request) -> tuple[float, float]:
+                prefill = self.latency.prefill_seconds([max(req.remaining_prefill, 1)])
+                midpoint_context = req.prompt_tokens + max(req.output_tokens // 2, 1)
+                decode = req.output_tokens * self.latency.decode_step_seconds([midpoint_context])
+                return (req.deadline_time - self.now - prefill - decode, req.arrival_time)
+
+            return sorted(self.waiting, key=slack)
+        return sorted(self.waiting, key=lambda r: r.arrival_time)
+
+    def _record_timeline(self, force: bool = False) -> None:
+        # Keep result payload bounded. This is display telemetry, not the event log.
+        total_target = max(self.cfg.timeline_points, 20)
+        if not force and len(self.timeline) >= total_target:
+            stride = max(2, len(self.timeline) // total_target + 1)
+            self.timeline = self.timeline[::stride]
+        kv_used = self.kv.used_gb(self.active, self.prefill_pending)
+        self.peak_kv_gb = max(self.peak_kv_gb, kv_used)
+        point = TimelinePoint(
+            time_s=self.now,
+            waiting=len(self.waiting),
+            prefill_pending=len(self.prefill_pending),
+            decoding=len(self.active),
+            completed=len(self.completed),
+            kv_used_gb=kv_used,
+            kv_capacity_gb=self.kv.capacity_gb,
+        )
+        if not self.timeline or force or self.now - self.timeline[-1].time_s >= max(self.cfg.duration_s / total_target, 0.05):
+            self.timeline.append(point)
+
+    def _advance(self, delta: float) -> None:
+        delta = max(delta, 0.0)
+        self.busy_time += delta
+        self.now += delta
+        self._admit_arrivals()
+        self._record_timeline()
+
+    def _idle_to_next_arrival(self) -> bool:
+        nxt = self._next_arrival()
+        if nxt is None:
+            return False
+        self.now = max(self.now, nxt)
+        self._admit_arrivals()
+        self._record_timeline()
+        return True
+
+    def _mark_complete(self) -> None:
+        done = [r for r in self.active if r.complete]
+        for r in done:
+            r.completion_time = self.now
+            self.completed.append(r)
+        if done:
+            done_ids = {r.request_id for r in done}
+            self.active = [r for r in self.active if r.request_id not in done_ids]
+
+    def _prefill_full_requests(self) -> bool:
+        slots = self.cfg.max_batch_size - len(self.active)
+        if slots <= 0 or not self.waiting:
+            return False
+        selected: list[Request] = []
+        total_tokens = 0
+        for req in self._waiting_sorted():
+            if len(selected) >= slots:
+                break
+            if selected and total_tokens + req.prompt_tokens > self.cfg.max_batch_tokens:
+                continue
+            if not self.kv.can_admit(req, self.active, selected):
+                continue
+            selected.append(req)
+            total_tokens += req.prompt_tokens
+
+        if not selected:
+            return False
+        selected_ids = {r.request_id for r in selected}
+        self.waiting = [r for r in self.waiting if r.request_id not in selected_ids]
+        for req in selected:
+            if req.first_prefill_time is None:
+                req.first_prefill_time = self.now
+        self._advance(self.latency.prefill_seconds([r.prompt_tokens for r in selected]))
+        for req in selected:
+            req.remaining_prefill = 0
+            self.active.append(req)
+        return True
+
+    def _prefill_chunked(self) -> bool:
+        slots = self.cfg.max_batch_size - len(self.active) - len(self.prefill_pending)
+        if slots > 0 and self.waiting:
+            for req in self._waiting_sorted():
+                if slots <= 0:
+                    break
+                if not self.kv.can_admit(req, self.active, self.prefill_pending):
+                    continue
+                self.waiting.remove(req)
+                if req.first_prefill_time is None:
+                    req.first_prefill_time = self.now
+                self.prefill_pending.append(req)
+                slots -= 1
+
+        if not self.prefill_pending:
+            return False
+
+        chunks: list[int] = []
+        selected: list[Request] = []
+        token_budget = self.cfg.max_batch_tokens
+        for req in list(self.prefill_pending):
+            if token_budget <= 0:
+                break
+            chunk = min(req.remaining_prefill, self.cfg.chunk_size, token_budget)
+            if chunk <= 0:
+                continue
+            selected.append(req)
+            chunks.append(chunk)
+            token_budget -= chunk
+
+        if not selected:
+            return False
+
+        # One prefill chunk. Decode is serviced on the next loop iteration,
+        # producing the intended prefill/decode interleaving.
+        self._advance(self.latency.prefill_seconds(chunks))
+        for req, chunk in zip(selected, chunks):
+            req.remaining_prefill -= chunk
+            if req.remaining_prefill <= 0:
+                self.prefill_pending.remove(req)
+                self.active.append(req)
+        return True
+
+    def _decode_step(self) -> bool:
+        if not self.active:
+            return False
+        contexts = [r.context_tokens for r in self.active]
+        self._advance(self.latency.decode_step_seconds(contexts))
+        for req in self.active:
+            req.generated_tokens += 1
+            if req.first_token_time is None:
+                req.first_token_time = self.now
+        self._mark_complete()
+        return True
+
+    def _run_static(self) -> None:
+        # Static batching deliberately refuses new admission while a batch is
+        # decoding. New arrivals queue until every member of the current batch
+        # completes, giving a clean baseline against continuous batching.
+        while len(self.completed) < len(self.requests):
+            self._admit_arrivals()
+            if not self.active:
+                if not self.waiting and not self._idle_to_next_arrival():
+                    break
+                selected = self._waiting_sorted()[: self.cfg.max_batch_size]
+                admitted: list[Request] = []
+                for req in selected:
+                    if self.kv.can_admit(req, admitted, None):
+                        admitted.append(req)
+                if not admitted:
+                    self.warnings.append("No static batch could fit in the configured KV budget.")
+                    break
+                ids = {r.request_id for r in admitted}
+                self.waiting = [r for r in self.waiting if r.request_id not in ids]
+                for req in admitted:
+                    req.first_prefill_time = self.now
+                self._advance(self.latency.prefill_seconds([r.prompt_tokens for r in admitted]))
+                for req in admitted:
+                    req.remaining_prefill = 0
+                    self.active.append(req)
+
+            # Finish this batch without admitting queued work into free slots.
+            while self.active:
+                contexts = [r.context_tokens for r in self.active]
+                delta = self.latency.decode_step_seconds(contexts)
+                self.busy_time += delta
+                self.now += delta
+                # Arrivals are queued but never admitted until the batch drains.
+                self._admit_arrivals()
+                for req in self.active:
+                    req.generated_tokens += 1
+                    if req.first_token_time is None:
+                        req.first_token_time = self.now
+                self._mark_complete()
+                self._record_timeline()
+
+    def _run_continuous(self) -> None:
+        while len(self.completed) < len(self.requests):
+            self._admit_arrivals()
+            progressed = False
+
+            if self.cfg.scheduler == "chunked_slo":
+                # Decode first if work is active, then execute one prefill chunk.
+                # This prevents long prompts from monopolizing the device.
+                if self.active:
+                    progressed = self._decode_step() or progressed
+                progressed = self._prefill_chunked() or progressed
+            else:
+                progressed = self._prefill_full_requests() or progressed
+                progressed = self._decode_step() or progressed
+
+            if not progressed:
+                if self.waiting or self.prefill_pending:
+                    self.warnings.append(
+                        "Simulation stalled: queued requests could not fit within the configured KV budget."
+                    )
+                    break
+                if not self._idle_to_next_arrival():
+                    break
+
+    def run(self) -> SimulationResult:
+        if not self.requests:
+            self.warnings.append("The workload generator produced zero requests; increase duration or request rate.")
+        self._record_timeline(force=True)
+        if self.cfg.scheduler == "static_fcfs":
+            self._run_static()
+        else:
+            self._run_continuous()
+        self._record_timeline(force=True)
+
+        makespan = max(self.now, self.cfg.duration_s if self.requests else 0.0)
+        summary, latency = summarize(self.completed, self.cfg, makespan, self.busy_time)
+        summary["requests_generated"] = len(self.requests)
+        summary["requests_unfinished"] = len(self.requests) - len(self.completed)
+
+        resource = {
+            "model_weight_gb": self.latency.model_weight_gb,
+            "kv_capacity_gb": self.kv.capacity_gb,
+            "peak_kv_gb": self.peak_kv_gb,
+            "peak_kv_utilization": self.peak_kv_gb / self.kv.capacity_gb if self.kv.capacity_gb > 0 else 0.0,
+            "accelerator_vram_gb": self.accelerator.vram_gb,
+        }
+
+        request_rows = []
+        # Preserve a bounded sample for scatterplots/export. Aggregate metrics
+        # still cover every completed request.
+        for req in self.completed[:2000]:
+            request_rows.append({
+                "request_id": req.request_id,
+                "arrival_time": req.arrival_time,
+                "prompt_tokens": req.prompt_tokens,
+                "output_tokens": req.output_tokens,
+                "ttft_ms": (req.first_token_time - req.arrival_time) * 1000.0 if req.first_token_time is not None else None,
+                "e2e_ms": (req.completion_time - req.arrival_time) * 1000.0 if req.completion_time is not None else None,
+            })
+
+        provenance = {
+            "simulator": "InferScale-Sim",
+            "version": "0.1.0",
+            "latency_profile_type": "analytical-reference",
+            "profile_warning": "Reference profiles are analytical proxies, not measured hardware benchmarks.",
+            "model_profile_source": self.model.source,
+            "accelerator_profile_source": self.accelerator.source,
+        }
+
+        return SimulationResult(
+            config=self.cfg.to_dict(),
+            provenance=provenance,
+            summary=summary,
+            latency=latency,
+            resource=resource,
+            requests=request_rows,
+            timeline=[asdict(p) for p in self.timeline],
+            warnings=self.warnings,
+        )
+
+
+def run_simulation(config: dict) -> dict:
+    return Simulator(SimulationConfig.from_dict(config)).run().to_dict()

py/inferscale/workloads.py

@@ -0,0 +1,69 @@
+from __future__ import annotations
+
+import math
+import random
+
+from .models import Request, SimulationConfig
+
+
+def _sample_lognormal(mean: float, cv: float, rng: random.Random, minimum: int = 1) -> int:
+    if cv <= 1e-9:
+        return max(minimum, int(round(mean)))
+    variance_ratio = cv * cv
+    sigma2 = math.log(1.0 + variance_ratio)
+    sigma = math.sqrt(sigma2)
+    mu = math.log(max(mean, 1e-6)) - sigma2 / 2.0
+    return max(minimum, int(round(rng.lognormvariate(mu, sigma))))
+
+
+def _arrival_times(cfg: SimulationConfig, rng: random.Random) -> list[float]:
+    rate = max(cfg.request_rate_rps, 1e-9)
+    arrivals: list[float] = []
+    t = 0.0
+
+    if cfg.arrival_process == "constant":
+        step = 1.0 / rate
+        while t < cfg.duration_s:
+            arrivals.append(t)
+            t += step
+        return arrivals
+
+    if cfg.arrival_process == "bursty":
+        # Alternating baseline/high-load windows. The mean offered load is not
+        # forced to equal request_rate_rps; the UI labels this clearly as a burst
+        # stress test rather than an average-rate generator.
+        while t < cfg.duration_s:
+            phase = int(t // max(cfg.burst_period_s, 0.1)) % 2
+            local_rate = rate * (cfg.burst_multiplier if phase else 0.55)
+            t += rng.expovariate(max(local_rate, 1e-9))
+            if t < cfg.duration_s:
+                arrivals.append(t)
+        return arrivals
+
+    if cfg.arrival_process != "poisson":
+        raise ValueError(f"Unknown arrival process: {cfg.arrival_process}")
+
+    while t < cfg.duration_s:
+        t += rng.expovariate(rate)
+        if t < cfg.duration_s:
+            arrivals.append(t)
+    return arrivals
+
+
+def generate_workload(cfg: SimulationConfig) -> list[Request]:
+    rng = random.Random(cfg.seed)
+    requests: list[Request] = []
+    for idx, arrival in enumerate(_arrival_times(cfg, rng)):
+        prompt = _sample_lognormal(cfg.prompt_tokens_mean, cfg.prompt_tokens_cv, rng)
+        output = _sample_lognormal(cfg.output_tokens_mean, cfg.output_tokens_cv, rng)
+        requests.append(
+            Request(
+                request_id=idx,
+                arrival_time=arrival,
+                prompt_tokens=prompt,
+                output_tokens=output,
+                deadline_time=arrival + cfg.slo_e2e_ms / 1000.0,
+                remaining_prefill=prompt,
+            )
+        )
+    return requests

pyproject.toml

@@ -0,0 +1,31 @@
+[build-system]
+requires = ["setuptools>=68"]
+build-backend = "setuptools.build_meta"
+
+[project]
+name = "inferscale-sim"
+version = "0.1.0"
+description = "Interactive LLM serving simulator and SLO capacity planner"
+readme = "README.md"
+requires-python = ">=3.10"
+license = {text = "MIT"}
+authors = [{name = "Archit Sharma"}]
+dependencies = []
+
+[project.optional-dependencies]
+dev = ["pytest>=8", "ruff>=0.9"]
+
+[tool.setuptools.packages.find]
+where = ["src"]
+
+[tool.pytest.ini_options]
+pythonpath = ["src"]
+testpaths = ["tests"]
+
+[tool.ruff]
+line-length = 120
+target-version = "py310"
+
+[tool.ruff.lint]
+select = ["E", "F", "I", "B", "UP"]
+ignore = ["E501"]

scripts/release_check.py

@@ -0,0 +1,69 @@
+from __future__ import annotations
+
+import re
+import sys
+from pathlib import Path
+
+ROOT = Path(__file__).resolve().parents[1]
+README = (ROOT / "README.md").read_text()
+SRC = ROOT / "src"
+sys.path.insert(0, str(SRC))
+
+from inferscale import run_simulation  # noqa: E402
+
+errors: list[str] = []
+
+match = re.search(r"^short_description:\s*(.+)$", README, re.MULTILINE)
+if not match:
+    errors.append("README metadata is missing short_description")
+    short = ""
+else:
+    short = match.group(1).strip().strip('"\'')
+    if len(short) > 60:
+        errors.append(f"short_description is {len(short)} chars; HF limit is 60")
+
+if "sdk: static" not in README:
+    errors.append("README metadata must use sdk: static")
+if not (ROOT / "index.html").exists():
+    errors.append("index.html missing")
+if not (ROOT / "worker.mjs").exists():
+    errors.append("worker.mjs missing")
+
+src_files = sorted((ROOT / "src" / "inferscale").glob("*.py"))
+web_files = sorted((ROOT / "py" / "inferscale").glob("*.py"))
+if [p.name for p in src_files] != [p.name for p in web_files]:
+    errors.append("browser Python mirror is stale; run python scripts/sync_web_python.py")
+else:
+    for src, web in zip(src_files, web_files):
+        if src.read_bytes() != web.read_bytes():
+            errors.append(f"browser mirror differs for {src.name}; run sync_web_python.py")
+
+smoke_cfg = {
+    "model": "Qwen2.5-3B",
+    "accelerator": "L4",
+    "quantization": "int8",
+    "duration_s": 4,
+    "request_rate_rps": 1,
+    "prompt_tokens_mean": 128,
+    "output_tokens_mean": 8,
+}
+try:
+    smoke = run_simulation(smoke_cfg)
+    if smoke["summary"]["requests_completed"] <= 0:
+        errors.append("simulation smoke test completed zero requests")
+    if smoke["provenance"]["latency_profile_type"] != "analytical-reference":
+        errors.append("profile provenance guard is missing")
+except Exception as exc:  # pragma: no cover - release diagnostic
+    errors.append(f"simulation smoke test raised: {exc}")
+
+if errors:
+    print("InferScale release check: FAIL")
+    for error in errors:
+        print(f"- {error}")
+    raise SystemExit(1)
+
+print("InferScale release check: PASS")
+print(f"HF short_description: {len(short)}/60 characters")
+print(f"Python modules mirrored: {len(src_files)}")
+print(f"Smoke requests completed: {smoke['summary']['requests_completed']}")
+print(f"Profile provenance: {smoke['provenance']['latency_profile_type']}")

scripts/run_simulation.py

@@ -0,0 +1,30 @@
+from __future__ import annotations
+
+import argparse
+import json
+from pathlib import Path
+
+from inferscale import run_simulation
+
+
+def main() -> None:
+    parser = argparse.ArgumentParser(description="Run an InferScale simulation locally.")
+    parser.add_argument("--config", type=Path, help="Optional JSON configuration file")
+    parser.add_argument("--output", type=Path, help="Write result JSON to this path")
+    args = parser.parse_args()
+
+    config = {}
+    if args.config:
+        config = json.loads(args.config.read_text())
+    result = run_simulation(config)
+    payload = json.dumps(result, indent=2)
+    if args.output:
+        args.output.parent.mkdir(parents=True, exist_ok=True)
+        args.output.write_text(payload)
+        print(f"Wrote {args.output}")
+    else:
+        print(payload)
+
+
+if __name__ == "__main__":
+    main()

scripts/sync_web_python.py

@@ -0,0 +1,14 @@
+from __future__ import annotations
+
+import shutil
+from pathlib import Path
+
+ROOT = Path(__file__).resolve().parents[1]
+SRC = ROOT / "src" / "inferscale"
+DEST = ROOT / "py" / "inferscale"
+
+if DEST.exists():
+    shutil.rmtree(DEST)
+DEST.parent.mkdir(parents=True, exist_ok=True)
+shutil.copytree(SRC, DEST, ignore=shutil.ignore_patterns("__pycache__", "*.pyc"))
+print(f"Synced {SRC} -> {DEST}")

src/inferscale/__init__.py

@@ -0,0 +1,15 @@
+from .api import execute, metadata
+from .models import SimulationConfig
+from .optimizer import capacity_search, compare_schedulers
+from .simulator import run_simulation
+
+__all__ = [
+    "SimulationConfig",
+    "capacity_search",
+    "compare_schedulers",
+    "execute",
+    "metadata",
+    "run_simulation",
+]
+
+__version__ = "0.1.0"

src/inferscale/api.py

@@ -0,0 +1,37 @@
+from __future__ import annotations
+
+from .optimizer import capacity_search, compare_schedulers
+from .profiles import ACCELERATORS, MODELS
+from .simulator import SCHEDULERS, run_simulation
+
+
+def metadata() -> dict:
+    return {
+        "version": "0.1.0",
+        "models": list(MODELS.keys()),
+        "accelerators": list(ACCELERATORS.keys()),
+        "schedulers": sorted(SCHEDULERS),
+        "profile_type": "analytical-reference",
+    }
+
+
+def execute(action: str, payload: dict) -> dict:
+    if action == "simulate":
+        return run_simulation(payload)
+    if action == "capacity":
+        config = payload.get("config", payload)
+        return capacity_search(
+            config,
+            min_rate=float(payload.get("min_rate", 0.25)),
+            max_rate=float(payload.get("max_rate", 32.0)),
+            iterations=int(payload.get("iterations", 8)),
+            repetitions=int(payload.get("repetitions", 2)),
+            headroom=float(payload.get("headroom", 0.20)),
+        )
+    if action == "compare":
+        config = payload.get("config", payload)
+        schedulers = payload.get("schedulers")
+        return compare_schedulers(config, schedulers)
+    if action == "metadata":
+        return metadata()
+    raise ValueError(f"Unknown action: {action}")

src/inferscale/kv_cache.py

@@ -0,0 +1,39 @@
+from __future__ import annotations
+
+import math
+
+from .latency import AnalyticalLatencyModel
+from .models import Request, SimulationConfig
+
+
+class KVCacheModel:
+    def __init__(self, latency_model: AnalyticalLatencyModel, cfg: SimulationConfig):
+        self.latency_model = latency_model
+        self.cfg = cfg
+        self.model_weight_gb = latency_model.model_weight_gb
+        total_vram = latency_model.accelerator.vram_gb
+        # Reserve non-KV space for runtime workspace and weights.
+        remaining = max(0.0, total_vram - self.model_weight_gb - 1.2)
+        self.capacity_gb = remaining * cfg.kv_memory_fraction
+
+    def _allocated_tokens(self, req: Request, include_output_reservation: bool = False) -> int:
+        live_tokens = req.prompt_tokens + req.generated_tokens
+        if self.cfg.scheduler == "static_fcfs" or include_output_reservation:
+            # Static baseline reserves the full sequence, illustrating memory
+            # fragmentation / over-reservation versus paged allocation.
+            return req.prompt_tokens + req.output_tokens
+        block = max(self.cfg.kv_block_tokens, 1)
+        return int(math.ceil(live_tokens / block) * block)
+
+    def request_gb(self, req: Request, include_output_reservation: bool = False) -> float:
+        tokens = self._allocated_tokens(req, include_output_reservation)
+        return tokens * self.latency_model.kv_bytes_per_token() / 1e9
+
+    def used_gb(self, active: list[Request], prefill_pending: list[Request] | None = None) -> float:
+        requests = list(active)
+        if prefill_pending:
+            requests.extend(prefill_pending)
+        return sum(self.request_gb(r) for r in requests)
+
+    def can_admit(self, req: Request, active: list[Request], prefill_pending: list[Request] | None = None) -> bool:
+        return self.used_gb(active, prefill_pending) + self.request_gb(req) <= self.capacity_gb + 1e-12

src/inferscale/latency.py

@@ -0,0 +1,106 @@
+from __future__ import annotations
+
+import math
+
+from .models import AcceleratorProfile, ModelProfile
+from .profiles import QUANTIZATION_BYTES, QUANTIZATION_COMPUTE_MULTIPLIER
+
+
+class AnalyticalLatencyModel:
+    """Hardware-aware analytical proxy for prefill/decode latency.
+
+    This model intentionally does *not* claim benchmark-grade accuracy. It uses
+    model architecture, accelerator peak specs, conservative efficiency factors,
+    and a roofline-style max(compute_time, memory_time) approximation. Public
+    profiles are tagged `analytical-reference` throughout the app.
+    """
+
+    def __init__(self, model: ModelProfile, accelerator: AcceleratorProfile, quantization: str = "fp16"):
+        if quantization not in QUANTIZATION_BYTES:
+            raise ValueError(f"Unsupported quantization: {quantization}")
+        self.model = model
+        self.accelerator = accelerator
+        self.quantization = quantization
+        self.weight_bytes_per_param = QUANTIZATION_BYTES[quantization]
+        self.compute_overhead = QUANTIZATION_COMPUTE_MULTIPLIER[quantization]
+
+    @property
+    def model_weight_gb(self) -> float:
+        return self.model.params_b * self.weight_bytes_per_param
+
+    def kv_bytes_per_token(self) -> float:
+        # K + V, all layers, KV heads only. KV state is assumed fp16 in v0.1.
+        return (
+            2
+            * self.model.layers
+            * self.model.kv_heads
+            * self.model.head_dim
+            * 2.0
+        )
+
+    def _compute_efficiency(self, batch_size: int, tokens: int) -> float:
+        scale = 1.0 + 0.12 * math.log2(max(batch_size, 1)) + 0.035 * math.log2(max(tokens, 1))
+        return min(0.88, self.accelerator.compute_efficiency * scale)
+
+    def _bandwidth_efficiency(self, batch_size: int) -> float:
+        scale = 1.0 + 0.06 * math.log2(max(batch_size, 1))
+        return min(0.91, self.accelerator.bandwidth_efficiency * scale)
+
+    def prefill_seconds(self, token_counts: list[int]) -> float:
+        if not token_counts:
+            return 0.0
+        batch = len(token_counts)
+        total_tokens = sum(token_counts)
+        max_seq = max(token_counts)
+
+        dense_flops = 2.0 * self.model.params_b * 1e9 * total_tokens
+        # Approximate quadratic attention component. It is small for short
+        # contexts but becomes visible at long prompts.
+        attention_flops = (
+            4.0
+            * self.model.layers
+            * self.model.hidden_size
+            * sum(t * t for t in token_counts)
+        )
+        flops = (dense_flops + attention_flops) * self.compute_overhead
+        compute = flops / (
+            self.accelerator.peak_tflops_fp16 * 1e12 * self._compute_efficiency(batch, max_seq)
+        )
+
+        weight_bytes = self.model.params_b * 1e9 * self.weight_bytes_per_param
+        activation_bytes = total_tokens * self.model.hidden_size * self.model.layers * 2.0 * 0.18
+        memory = (weight_bytes + activation_bytes) / (
+            self.accelerator.bandwidth_gbps * 1e9 * self._bandwidth_efficiency(batch)
+        )
+        # Kernel launch / scheduling proxy prevents implausibly tiny times.
+        launch = 0.0018 + 0.00008 * batch
+        return max(compute, memory * 0.28) + launch
+
+    def decode_step_seconds(self, context_lengths: list[int]) -> float:
+        if not context_lengths:
+            return 0.0
+        batch = len(context_lengths)
+        avg_context = sum(context_lengths) / batch
+
+        dense_flops = 2.0 * self.model.params_b * 1e9 * batch
+        attention_flops = (
+            4.0
+            * self.model.layers
+            * self.model.hidden_size
+            * sum(context_lengths)
+        )
+        flops = (dense_flops + attention_flops) * self.compute_overhead
+        compute = flops / (
+            self.accelerator.peak_tflops_fp16 * 1e12 * self._compute_efficiency(batch, 1)
+        )
+
+        # Decode is commonly memory-bound. We model one shared weight stream plus
+        # KV reads that scale with batch and context length.
+        weight_bytes = self.model.params_b * 1e9 * self.weight_bytes_per_param
+        kv_read_bytes = self.kv_bytes_per_token() * sum(context_lengths)
+        memory = (weight_bytes + kv_read_bytes) / (
+            self.accelerator.bandwidth_gbps * 1e9 * self._bandwidth_efficiency(batch)
+        )
+
+        launch = 0.0012 + 0.000035 * batch + 0.00000003 * avg_context
+        return max(compute, memory) + launch

src/inferscale/metrics.py

@@ -0,0 +1,79 @@
+from __future__ import annotations
+
+import math
+from statistics import mean
+
+from .models import Request, RequestMetrics, SimulationConfig
+
+
+def percentile(values: list[float], q: float) -> float:
+    if not values:
+        return 0.0
+    vals = sorted(values)
+    if len(vals) == 1:
+        return vals[0]
+    pos = (len(vals) - 1) * q
+    lo = math.floor(pos)
+    hi = math.ceil(pos)
+    if lo == hi:
+        return vals[lo]
+    frac = pos - lo
+    return vals[lo] * (1.0 - frac) + vals[hi] * frac
+
+
+def request_metrics(req: Request, cfg: SimulationConfig) -> RequestMetrics:
+    if req.first_token_time is None or req.completion_time is None:
+        raise ValueError("Request is incomplete")
+    ttft_ms = (req.first_token_time - req.arrival_time) * 1000.0
+    e2e_ms = (req.completion_time - req.arrival_time) * 1000.0
+    if req.output_tokens <= 1:
+        tpot_ms = 0.0
+    else:
+        tpot_ms = (req.completion_time - req.first_token_time) * 1000.0 / (req.output_tokens - 1)
+    prefill_start = req.first_prefill_time if req.first_prefill_time is not None else req.arrival_time
+    queue_ms = max(0.0, (prefill_start - req.arrival_time) * 1000.0)
+    met_ttft = ttft_ms <= cfg.slo_ttft_ms
+    met_e2e = e2e_ms <= cfg.slo_e2e_ms
+    return RequestMetrics(
+        request_id=req.request_id,
+        arrival_time=req.arrival_time,
+        prompt_tokens=req.prompt_tokens,
+        output_tokens=req.output_tokens,
+        ttft_ms=ttft_ms,
+        e2e_ms=e2e_ms,
+        tpot_ms=tpot_ms,
+        queue_ms=queue_ms,
+        met_ttft_slo=met_ttft,
+        met_e2e_slo=met_e2e,
+        met_all_slos=met_ttft and met_e2e,
+    )
+
+
+def summarize(completed: list[Request], cfg: SimulationConfig, makespan_s: float, busy_time_s: float) -> tuple[dict, dict]:
+    metrics = [request_metrics(r, cfg) for r in completed]
+    ttft = [m.ttft_ms for m in metrics]
+    e2e = [m.e2e_ms for m in metrics]
+    tpot = [m.tpot_ms for m in metrics]
+    queue = [m.queue_ms for m in metrics]
+    total_output = sum(r.output_tokens for r in completed)
+    met = sum(m.met_all_slos for m in metrics)
+    duration = max(makespan_s, 1e-9)
+
+    summary = {
+        "requests_completed": len(completed),
+        "request_throughput_rps": len(completed) / duration,
+        "output_throughput_tps": total_output / duration,
+        "goodput_rps": met / duration,
+        "slo_attainment": met / len(metrics) if metrics else 0.0,
+        "simulated_makespan_s": makespan_s,
+        "busy_fraction": min(1.0, busy_time_s / duration),
+        "mean_prompt_tokens": mean([r.prompt_tokens for r in completed]) if completed else 0.0,
+        "mean_output_tokens": mean([r.output_tokens for r in completed]) if completed else 0.0,
+    }
+    latency = {
+        "ttft_ms": {"p50": percentile(ttft, 0.50), "p95": percentile(ttft, 0.95), "p99": percentile(ttft, 0.99)},
+        "e2e_ms": {"p50": percentile(e2e, 0.50), "p95": percentile(e2e, 0.95), "p99": percentile(e2e, 0.99)},
+        "tpot_ms": {"p50": percentile(tpot, 0.50), "p95": percentile(tpot, 0.95), "p99": percentile(tpot, 0.99)},
+        "queue_ms": {"p50": percentile(queue, 0.50), "p95": percentile(queue, 0.95), "p99": percentile(queue, 0.99)},
+    }
+    return summary, latency

src/inferscale/models.py

@@ -0,0 +1,130 @@
+from __future__ import annotations
+
+from dataclasses import asdict, dataclass, field
+from typing import Any
+
+
+@dataclass
+class ModelProfile:
+    name: str
+    params_b: float
+    layers: int
+    hidden_size: int
+    attention_heads: int
+    kv_heads: int
+    default_dtype_bytes: float = 2.0
+    source: str = "analytical-reference"
+
+    @property
+    def head_dim(self) -> int:
+        return self.hidden_size // self.attention_heads
+
+
+@dataclass
+class AcceleratorProfile:
+    name: str
+    vram_gb: float
+    peak_tflops_fp16: float
+    bandwidth_gbps: float
+    compute_efficiency: float
+    bandwidth_efficiency: float
+    source: str = "vendor-spec-reference"
+
+
+@dataclass
+class SimulationConfig:
+    model: str = "Llama-3.1-8B"
+    accelerator: str = "L4"
+    scheduler: str = "continuous_fcfs"
+    arrival_process: str = "poisson"
+    request_rate_rps: float = 4.0
+    duration_s: float = 60.0
+    prompt_tokens_mean: int = 512
+    prompt_tokens_cv: float = 0.35
+    output_tokens_mean: int = 128
+    output_tokens_cv: float = 0.35
+    max_batch_size: int = 16
+    max_batch_tokens: int = 8192
+    chunk_size: int = 512
+    kv_block_tokens: int = 16
+    kv_memory_fraction: float = 0.85
+    quantization: str = "fp16"
+    seed: int = 7
+    slo_ttft_ms: float = 500.0
+    slo_e2e_ms: float = 8000.0
+    slo_attainment_target: float = 0.99
+    burst_multiplier: float = 3.0
+    burst_period_s: float = 10.0
+    timeline_points: int = 300
+
+    @classmethod
+    def from_dict(cls, data: dict[str, Any]) -> "SimulationConfig":
+        allowed = cls.__dataclass_fields__.keys()
+        return cls(**{k: data[k] for k in allowed if k in data})
+
+    def to_dict(self) -> dict[str, Any]:
+        return asdict(self)
+
+
+@dataclass
+class Request:
+    request_id: int
+    arrival_time: float
+    prompt_tokens: int
+    output_tokens: int
+    deadline_time: float
+    remaining_prefill: int
+    generated_tokens: int = 0
+    first_prefill_time: float | None = None
+    first_token_time: float | None = None
+    completion_time: float | None = None
+    priority: int = 0
+
+    @property
+    def context_tokens(self) -> int:
+        return self.prompt_tokens + self.generated_tokens
+
+    @property
+    def complete(self) -> bool:
+        return self.generated_tokens >= self.output_tokens
+
+
+@dataclass
+class RequestMetrics:
+    request_id: int
+    arrival_time: float
+    prompt_tokens: int
+    output_tokens: int
+    ttft_ms: float
+    e2e_ms: float
+    tpot_ms: float
+    queue_ms: float
+    met_ttft_slo: bool
+    met_e2e_slo: bool
+    met_all_slos: bool
+
+
+@dataclass
+class TimelinePoint:
+    time_s: float
+    waiting: int
+    prefill_pending: int
+    decoding: int
+    completed: int
+    kv_used_gb: float
+    kv_capacity_gb: float
+
+
+@dataclass
+class SimulationResult:
+    config: dict[str, Any]
+    provenance: dict[str, Any]
+    summary: dict[str, Any]
+    latency: dict[str, Any]
+    resource: dict[str, Any]
+    requests: list[dict[str, Any]] = field(default_factory=list)
+    timeline: list[dict[str, Any]] = field(default_factory=list)
+    warnings: list[str] = field(default_factory=list)
+
+    def to_dict(self) -> dict[str, Any]:
+        return asdict(self)

src/inferscale/optimizer.py

@@ -0,0 +1,106 @@
+from __future__ import annotations
+
+from copy import deepcopy
+from statistics import mean
+
+from .models import SimulationConfig
+from .simulator import run_simulation
+
+
+def evaluate_rate(base: SimulationConfig, rate: float, repetitions: int = 3) -> dict:
+    runs = []
+    for rep in range(repetitions):
+        cfg = deepcopy(base)
+        cfg.request_rate_rps = rate
+        cfg.seed = base.seed + rep * 101
+        runs.append(run_simulation(cfg.to_dict()))
+    attainment = mean(r["summary"]["slo_attainment"] for r in runs)
+    goodput = mean(r["summary"]["goodput_rps"] for r in runs)
+    p95_ttft = mean(r["latency"]["ttft_ms"]["p95"] for r in runs)
+    p95_e2e = mean(r["latency"]["e2e_ms"]["p95"] for r in runs)
+    unfinished = mean(r["summary"]["requests_unfinished"] for r in runs)
+    passed = attainment >= base.slo_attainment_target and unfinished <= 0.0
+    return {
+        "rate_rps": rate,
+        "passed": passed,
+        "slo_attainment": attainment,
+        "goodput_rps": goodput,
+        "p95_ttft_ms": p95_ttft,
+        "p95_e2e_ms": p95_e2e,
+        "mean_unfinished": unfinished,
+    }
+
+
+def capacity_search(config: dict, min_rate: float = 0.25, max_rate: float = 32.0, iterations: int = 8, repetitions: int = 2, headroom: float = 0.20) -> dict:
+    base = SimulationConfig.from_dict(config)
+    low = max(0.01, min_rate)
+    high = max(low * 1.01, max_rate)
+    trace: list[dict] = []
+
+    low_eval = evaluate_rate(base, low, repetitions)
+    trace.append(low_eval)
+    if not low_eval["passed"]:
+        return {
+            "status": "no_feasible_rate",
+            "capacity_rps": 0.0,
+            "recommended_rps": 0.0,
+            "trace": trace,
+        }
+
+    high_eval = evaluate_rate(base, high, repetitions)
+    trace.append(high_eval)
+    if high_eval["passed"]:
+        capacity = high
+        return {
+            "status": "upper_bound_still_feasible",
+            "capacity_rps": capacity,
+            "recommended_rps": capacity * (1.0 - headroom),
+            "trace": sorted(trace, key=lambda x: x["rate_rps"]),
+        }
+
+    best = low
+    for _ in range(max(iterations, 1)):
+        mid = (low + high) / 2.0
+        result = evaluate_rate(base, mid, repetitions)
+        trace.append(result)
+        if result["passed"]:
+            best = mid
+            low = mid
+        else:
+            high = mid
+
+    return {
+        "status": "ok",
+        "capacity_rps": best,
+        "recommended_rps": best * (1.0 - headroom),
+        "headroom": headroom,
+        "trace": sorted(trace, key=lambda x: x["rate_rps"]),
+    }
+
+
+def compare_schedulers(config: dict, schedulers: list[str] | None = None) -> dict:
+    schedulers = schedulers or [
+        "static_fcfs",
+        "continuous_fcfs",
+        "continuous_sjf",
+        "continuous_slo",
+        "chunked_slo",
+    ]
+    base = SimulationConfig.from_dict(config)
+    rows = []
+    for scheduler in schedulers:
+        cfg = deepcopy(base)
+        cfg.scheduler = scheduler
+        result = run_simulation(cfg.to_dict())
+        rows.append({
+            "scheduler": scheduler,
+            "request_throughput_rps": result["summary"]["request_throughput_rps"],
+            "goodput_rps": result["summary"]["goodput_rps"],
+            "slo_attainment": result["summary"]["slo_attainment"],
+            "p95_ttft_ms": result["latency"]["ttft_ms"]["p95"],
+            "p95_e2e_ms": result["latency"]["e2e_ms"]["p95"],
+            "peak_kv_utilization": result["resource"]["peak_kv_utilization"],
+            "unfinished": result["summary"]["requests_unfinished"],
+        })
+    rows.sort(key=lambda r: (r["slo_attainment"], r["goodput_rps"]), reverse=True)
+    return {"rows": rows}

src/inferscale/profiles.py

@@ -0,0 +1,88 @@
+from __future__ import annotations
+
+from .models import AcceleratorProfile, ModelProfile
+
+
+MODELS: dict[str, ModelProfile] = {
+    "Llama-3.1-8B": ModelProfile(
+        name="Llama-3.1-8B",
+        params_b=8.03,
+        layers=32,
+        hidden_size=4096,
+        attention_heads=32,
+        kv_heads=8,
+    ),
+    "Mistral-7B-v0.3": ModelProfile(
+        name="Mistral-7B-v0.3",
+        params_b=7.25,
+        layers=32,
+        hidden_size=4096,
+        attention_heads=32,
+        kv_heads=8,
+    ),
+    "Qwen2.5-3B": ModelProfile(
+        name="Qwen2.5-3B",
+        params_b=3.09,
+        layers=36,
+        hidden_size=2048,
+        attention_heads=16,
+        kv_heads=2,
+    ),
+}
+
+
+ACCELERATORS: dict[str, AcceleratorProfile] = {
+    "L4": AcceleratorProfile(
+        name="NVIDIA L4",
+        vram_gb=24.0,
+        peak_tflops_fp16=121.0,
+        bandwidth_gbps=300.0,
+        compute_efficiency=0.40,
+        bandwidth_efficiency=0.72,
+    ),
+    "A10G": AcceleratorProfile(
+        name="NVIDIA A10G",
+        vram_gb=24.0,
+        peak_tflops_fp16=125.0,
+        bandwidth_gbps=600.0,
+        compute_efficiency=0.40,
+        bandwidth_efficiency=0.70,
+    ),
+    "A100-40GB": AcceleratorProfile(
+        name="NVIDIA A100 40GB",
+        vram_gb=40.0,
+        peak_tflops_fp16=312.0,
+        bandwidth_gbps=1555.0,
+        compute_efficiency=0.47,
+        bandwidth_efficiency=0.76,
+    ),
+}
+
+
+QUANTIZATION_BYTES = {
+    "fp16": 2.0,
+    "int8": 1.0,
+    "int4": 0.5,
+}
+
+# Compute dequantization / packing overheads are deliberately conservative analytical
+# modifiers, not empirical benchmark claims.
+QUANTIZATION_COMPUTE_MULTIPLIER = {
+    "fp16": 1.00,
+    "int8": 1.07,
+    "int4": 1.16,
+}
+
+
+def get_model(name: str) -> ModelProfile:
+    try:
+        return MODELS[name]
+    except KeyError as exc:
+        raise ValueError(f"Unknown model profile: {name}") from exc
+
+
+def get_accelerator(name: str) -> AcceleratorProfile:
+    try:
+        return ACCELERATORS[name]
+    except KeyError as exc:
+        raise ValueError(f"Unknown accelerator profile: {name}") from exc

src/inferscale/simulator.py

@@ -0,0 +1,321 @@
+from __future__ import annotations
+
+from dataclasses import asdict
+
+from .kv_cache import KVCacheModel
+from .latency import AnalyticalLatencyModel
+from .metrics import summarize
+from .models import Request, SimulationConfig, SimulationResult, TimelinePoint
+from .profiles import get_accelerator, get_model
+from .workloads import generate_workload
+
+
+SCHEDULERS = {
+    "static_fcfs",
+    "continuous_fcfs",
+    "continuous_sjf",
+    "continuous_slo",
+    "chunked_slo",
+}
+
+
+class Simulator:
+    def __init__(self, cfg: SimulationConfig):
+        if cfg.scheduler not in SCHEDULERS:
+            raise ValueError(f"Unsupported scheduler: {cfg.scheduler}")
+        self.cfg = cfg
+        self.model = get_model(cfg.model)
+        self.accelerator = get_accelerator(cfg.accelerator)
+        self.latency = AnalyticalLatencyModel(self.model, self.accelerator, cfg.quantization)
+        self.kv = KVCacheModel(self.latency, cfg)
+        self.requests = generate_workload(cfg)
+        self.pending_idx = 0
+        self.waiting: list[Request] = []
+        self.prefill_pending: list[Request] = []
+        self.active: list[Request] = []
+        self.completed: list[Request] = []
+        self.now = 0.0
+        self.busy_time = 0.0
+        self.peak_kv_gb = 0.0
+        self.timeline: list[TimelinePoint] = []
+        self.warnings: list[str] = []
+
+    def _admit_arrivals(self) -> None:
+        while self.pending_idx < len(self.requests) and self.requests[self.pending_idx].arrival_time <= self.now + 1e-12:
+            self.waiting.append(self.requests[self.pending_idx])
+            self.pending_idx += 1
+
+    def _next_arrival(self) -> float | None:
+        if self.pending_idx >= len(self.requests):
+            return None
+        return self.requests[self.pending_idx].arrival_time
+
+    def _waiting_sorted(self) -> list[Request]:
+        if self.cfg.scheduler == "continuous_sjf":
+            return sorted(self.waiting, key=lambda r: (r.prompt_tokens + r.output_tokens, r.arrival_time))
+        if self.cfg.scheduler in {"continuous_slo", "chunked_slo"}:
+            # Least-slack-first proxy: deadline minus an analytical estimate of
+            # remaining standalone service. Unlike plain EDF, this distinguishes
+            # requests with the same relative SLO but heterogeneous token lengths.
+            def slack(req: Request) -> tuple[float, float]:
+                prefill = self.latency.prefill_seconds([max(req.remaining_prefill, 1)])
+                midpoint_context = req.prompt_tokens + max(req.output_tokens // 2, 1)
+                decode = req.output_tokens * self.latency.decode_step_seconds([midpoint_context])
+                return (req.deadline_time - self.now - prefill - decode, req.arrival_time)
+
+            return sorted(self.waiting, key=slack)
+        return sorted(self.waiting, key=lambda r: r.arrival_time)
+
+    def _record_timeline(self, force: bool = False) -> None:
+        # Keep result payload bounded. This is display telemetry, not the event log.
+        total_target = max(self.cfg.timeline_points, 20)
+        if not force and len(self.timeline) >= total_target:
+            stride = max(2, len(self.timeline) // total_target + 1)
+            self.timeline = self.timeline[::stride]
+        kv_used = self.kv.used_gb(self.active, self.prefill_pending)
+        self.peak_kv_gb = max(self.peak_kv_gb, kv_used)
+        point = TimelinePoint(
+            time_s=self.now,
+            waiting=len(self.waiting),
+            prefill_pending=len(self.prefill_pending),
+            decoding=len(self.active),
+            completed=len(self.completed),
+            kv_used_gb=kv_used,
+            kv_capacity_gb=self.kv.capacity_gb,
+        )
+        if not self.timeline or force or self.now - self.timeline[-1].time_s >= max(self.cfg.duration_s / total_target, 0.05):
+            self.timeline.append(point)
+
+    def _advance(self, delta: float) -> None:
+        delta = max(delta, 0.0)
+        self.busy_time += delta
+        self.now += delta
+        self._admit_arrivals()
+        self._record_timeline()
+
+    def _idle_to_next_arrival(self) -> bool:
+        nxt = self._next_arrival()
+        if nxt is None:
+            return False
+        self.now = max(self.now, nxt)
+        self._admit_arrivals()
+        self._record_timeline()
+        return True
+
+    def _mark_complete(self) -> None:
+        done = [r for r in self.active if r.complete]
+        for r in done:
+            r.completion_time = self.now
+            self.completed.append(r)
+        if done:
+            done_ids = {r.request_id for r in done}
+            self.active = [r for r in self.active if r.request_id not in done_ids]
+
+    def _prefill_full_requests(self) -> bool:
+        slots = self.cfg.max_batch_size - len(self.active)
+        if slots <= 0 or not self.waiting:
+            return False
+        selected: list[Request] = []
+        total_tokens = 0
+        for req in self._waiting_sorted():
+            if len(selected) >= slots:
+                break
+            if selected and total_tokens + req.prompt_tokens > self.cfg.max_batch_tokens:
+                continue
+            if not self.kv.can_admit(req, self.active, selected):
+                continue
+            selected.append(req)
+            total_tokens += req.prompt_tokens
+
+        if not selected:
+            return False
+        selected_ids = {r.request_id for r in selected}
+        self.waiting = [r for r in self.waiting if r.request_id not in selected_ids]
+        for req in selected:
+            if req.first_prefill_time is None:
+                req.first_prefill_time = self.now
+        self._advance(self.latency.prefill_seconds([r.prompt_tokens for r in selected]))
+        for req in selected:
+            req.remaining_prefill = 0
+            self.active.append(req)
+        return True
+
+    def _prefill_chunked(self) -> bool:
+        slots = self.cfg.max_batch_size - len(self.active) - len(self.prefill_pending)
+        if slots > 0 and self.waiting:
+            for req in self._waiting_sorted():
+                if slots <= 0:
+                    break
+                if not self.kv.can_admit(req, self.active, self.prefill_pending):
+                    continue
+                self.waiting.remove(req)
+                if req.first_prefill_time is None:
+                    req.first_prefill_time = self.now
+                self.prefill_pending.append(req)
+                slots -= 1
+
+        if not self.prefill_pending:
+            return False
+
+        chunks: list[int] = []
+        selected: list[Request] = []
+        token_budget = self.cfg.max_batch_tokens
+        for req in list(self.prefill_pending):
+            if token_budget <= 0:
+                break
+            chunk = min(req.remaining_prefill, self.cfg.chunk_size, token_budget)
+            if chunk <= 0:
+                continue
+            selected.append(req)
+            chunks.append(chunk)
+            token_budget -= chunk
+
+        if not selected:
+            return False
+
+        # One prefill chunk. Decode is serviced on the next loop iteration,
+        # producing the intended prefill/decode interleaving.
+        self._advance(self.latency.prefill_seconds(chunks))
+        for req, chunk in zip(selected, chunks):
+            req.remaining_prefill -= chunk
+            if req.remaining_prefill <= 0:
+                self.prefill_pending.remove(req)
+                self.active.append(req)
+        return True
+
+    def _decode_step(self) -> bool:
+        if not self.active:
+            return False
+        contexts = [r.context_tokens for r in self.active]
+        self._advance(self.latency.decode_step_seconds(contexts))
+        for req in self.active:
+            req.generated_tokens += 1
+            if req.first_token_time is None:
+                req.first_token_time = self.now
+        self._mark_complete()
+        return True
+
+    def _run_static(self) -> None:
+        # Static batching deliberately refuses new admission while a batch is
+        # decoding. New arrivals queue until every member of the current batch
+        # completes, giving a clean baseline against continuous batching.
+        while len(self.completed) < len(self.requests):
+            self._admit_arrivals()
+            if not self.active:
+                if not self.waiting and not self._idle_to_next_arrival():
+                    break
+                selected = self._waiting_sorted()[: self.cfg.max_batch_size]
+                admitted: list[Request] = []
+                for req in selected:
+                    if self.kv.can_admit(req, admitted, None):
+                        admitted.append(req)
+                if not admitted:
+                    self.warnings.append("No static batch could fit in the configured KV budget.")
+                    break
+                ids = {r.request_id for r in admitted}
+                self.waiting = [r for r in self.waiting if r.request_id not in ids]
+                for req in admitted:
+                    req.first_prefill_time = self.now
+                self._advance(self.latency.prefill_seconds([r.prompt_tokens for r in admitted]))
+                for req in admitted:
+                    req.remaining_prefill = 0
+                    self.active.append(req)
+
+            # Finish this batch without admitting queued work into free slots.
+            while self.active:
+                contexts = [r.context_tokens for r in self.active]
+                delta = self.latency.decode_step_seconds(contexts)
+                self.busy_time += delta
+                self.now += delta
+                # Arrivals are queued but never admitted until the batch drains.
+                self._admit_arrivals()
+                for req in self.active:
+                    req.generated_tokens += 1
+                    if req.first_token_time is None:
+                        req.first_token_time = self.now
+                self._mark_complete()
+                self._record_timeline()
+
+    def _run_continuous(self) -> None:
+        while len(self.completed) < len(self.requests):
+            self._admit_arrivals()
+            progressed = False
+
+            if self.cfg.scheduler == "chunked_slo":
+                # Decode first if work is active, then execute one prefill chunk.
+                # This prevents long prompts from monopolizing the device.
+                if self.active:
+                    progressed = self._decode_step() or progressed
+                progressed = self._prefill_chunked() or progressed
+            else:
+                progressed = self._prefill_full_requests() or progressed
+                progressed = self._decode_step() or progressed
+
+            if not progressed:
+                if self.waiting or self.prefill_pending:
+                    self.warnings.append(
+                        "Simulation stalled: queued requests could not fit within the configured KV budget."
+                    )
+                    break
+                if not self._idle_to_next_arrival():
+                    break
+
+    def run(self) -> SimulationResult:
+        if not self.requests:
+            self.warnings.append("The workload generator produced zero requests; increase duration or request rate.")
+        self._record_timeline(force=True)
+        if self.cfg.scheduler == "static_fcfs":
+            self._run_static()
+        else:
+            self._run_continuous()
+        self._record_timeline(force=True)
+
+        makespan = max(self.now, self.cfg.duration_s if self.requests else 0.0)
+        summary, latency = summarize(self.completed, self.cfg, makespan, self.busy_time)
+        summary["requests_generated"] = len(self.requests)
+        summary["requests_unfinished"] = len(self.requests) - len(self.completed)
+
+        resource = {
+            "model_weight_gb": self.latency.model_weight_gb,
+            "kv_capacity_gb": self.kv.capacity_gb,
+            "peak_kv_gb": self.peak_kv_gb,
+            "peak_kv_utilization": self.peak_kv_gb / self.kv.capacity_gb if self.kv.capacity_gb > 0 else 0.0,
+            "accelerator_vram_gb": self.accelerator.vram_gb,
+        }
+
+        request_rows = []
+        # Preserve a bounded sample for scatterplots/export. Aggregate metrics
+        # still cover every completed request.
+        for req in self.completed[:2000]:
+            request_rows.append({
+                "request_id": req.request_id,
+                "arrival_time": req.arrival_time,
+                "prompt_tokens": req.prompt_tokens,
+                "output_tokens": req.output_tokens,
+                "ttft_ms": (req.first_token_time - req.arrival_time) * 1000.0 if req.first_token_time is not None else None,
+                "e2e_ms": (req.completion_time - req.arrival_time) * 1000.0 if req.completion_time is not None else None,
+            })
+
+        provenance = {
+            "simulator": "InferScale-Sim",
+            "version": "0.1.0",
+            "latency_profile_type": "analytical-reference",
+            "profile_warning": "Reference profiles are analytical proxies, not measured hardware benchmarks.",
+            "model_profile_source": self.model.source,
+            "accelerator_profile_source": self.accelerator.source,
+        }
+
+        return SimulationResult(
+            config=self.cfg.to_dict(),
+            provenance=provenance,
+            summary=summary,
+            latency=latency,
+            resource=resource,
+            requests=request_rows,
+            timeline=[asdict(p) for p in self.timeline],
+            warnings=self.warnings,
+        )
+
+
+def run_simulation(config: dict) -> dict:
+    return Simulator(SimulationConfig.from_dict(config)).run().to_dict()

src/inferscale/workloads.py

@@ -0,0 +1,69 @@
+from __future__ import annotations
+
+import math
+import random
+
+from .models import Request, SimulationConfig
+
+
+def _sample_lognormal(mean: float, cv: float, rng: random.Random, minimum: int = 1) -> int:
+    if cv <= 1e-9:
+        return max(minimum, int(round(mean)))
+    variance_ratio = cv * cv
+    sigma2 = math.log(1.0 + variance_ratio)
+    sigma = math.sqrt(sigma2)
+    mu = math.log(max(mean, 1e-6)) - sigma2 / 2.0
+    return max(minimum, int(round(rng.lognormvariate(mu, sigma))))
+
+
+def _arrival_times(cfg: SimulationConfig, rng: random.Random) -> list[float]:
+    rate = max(cfg.request_rate_rps, 1e-9)
+    arrivals: list[float] = []
+    t = 0.0
+
+    if cfg.arrival_process == "constant":
+        step = 1.0 / rate
+        while t < cfg.duration_s:
+            arrivals.append(t)
+            t += step
+        return arrivals
+
+    if cfg.arrival_process == "bursty":
+        # Alternating baseline/high-load windows. The mean offered load is not
+        # forced to equal request_rate_rps; the UI labels this clearly as a burst
+        # stress test rather than an average-rate generator.
+        while t < cfg.duration_s:
+            phase = int(t // max(cfg.burst_period_s, 0.1)) % 2
+            local_rate = rate * (cfg.burst_multiplier if phase else 0.55)
+            t += rng.expovariate(max(local_rate, 1e-9))
+            if t < cfg.duration_s:
+                arrivals.append(t)
+        return arrivals
+
+    if cfg.arrival_process != "poisson":
+        raise ValueError(f"Unknown arrival process: {cfg.arrival_process}")
+
+    while t < cfg.duration_s:
+        t += rng.expovariate(rate)
+        if t < cfg.duration_s:
+            arrivals.append(t)
+    return arrivals
+
+
+def generate_workload(cfg: SimulationConfig) -> list[Request]:
+    rng = random.Random(cfg.seed)
+    requests: list[Request] = []
+    for idx, arrival in enumerate(_arrival_times(cfg, rng)):
+        prompt = _sample_lognormal(cfg.prompt_tokens_mean, cfg.prompt_tokens_cv, rng)
+        output = _sample_lognormal(cfg.output_tokens_mean, cfg.output_tokens_cv, rng)
+        requests.append(
+            Request(
+                request_id=idx,
+                arrival_time=arrival,
+                prompt_tokens=prompt,
+                output_tokens=output,
+                deadline_time=arrival + cfg.slo_e2e_ms / 1000.0,
+                remaining_prefill=prompt,
+            )
+        )
+    return requests

styles.css

@@ -0,0 +1,79 @@
+:root {
+  --bg: #080b12;
+  --panel: #10151f;
+  --panel2: #151c28;
+  --line: #242d3c;
+  --text: #edf1f7;
+  --muted: #95a0b3;
+  --accent: #8b7cff;
+  --accent2: #55c2ff;
+  --good: #63d9a5;
+  --warn: #f5c76e;
+  --danger: #ff7b8a;
+  --shadow: 0 22px 70px rgba(0,0,0,.28);
+}
+* { box-sizing: border-box; }
+html { background: var(--bg); color-scheme: dark; }
+body { margin: 0; font-family: Inter, ui-sans-serif, system-ui, -apple-system, BlinkMacSystemFont, "Segoe UI", sans-serif; color: var(--text); background: radial-gradient(circle at 70% -10%, rgba(82, 80, 210, .17), transparent 30rem), var(--bg); }
+button, input, select { font: inherit; }
+.topbar { height: 76px; padding: 0 34px; border-bottom: 1px solid rgba(255,255,255,.06); display:flex; align-items:center; justify-content:space-between; position:sticky; top:0; z-index:20; background:rgba(8,11,18,.88); backdrop-filter: blur(16px); }
+.brand-wrap { display:flex; align-items:center; gap:13px; }
+.logo { width:38px; height:38px; display:grid; place-items:center; border-radius:10px; background:linear-gradient(135deg,var(--accent),var(--accent2)); color:white; font-weight:800; letter-spacing:-.04em; }
+.brand { font-weight:750; letter-spacing:-.02em; }
+.subtitle { color:var(--muted); font-size:12px; margin-top:2px; }
+.runtime-pill { display:flex; gap:8px; align-items:center; padding:8px 12px; border:1px solid var(--line); border-radius:99px; color:var(--muted); font-size:12px; background:rgba(255,255,255,.025); }
+.dot { width:7px; height:7px; border-radius:50%; background:var(--warn); box-shadow:0 0 14px currentColor; }
+.runtime-pill.ready .dot { background:var(--good); }
+.runtime-pill.error .dot { background:var(--danger); }
+.shell { width:min(1480px, calc(100% - 44px)); margin:0 auto; padding:42px 0 60px; }
+.hero { display:grid; grid-template-columns:minmax(0,1.45fr) minmax(400px,.75fr); gap:44px; align-items:end; padding:26px 4px 34px; }
+.eyebrow,.section-kicker { font-size:11px; letter-spacing:.12em; text-transform:uppercase; color:#9f96ff; font-weight:750; }
+h1 { font-size:clamp(34px,4vw,60px); line-height:1.03; letter-spacing:-.045em; max-width:940px; margin:10px 0 18px; }
+.hero p { max-width:830px; color:var(--muted); line-height:1.65; font-size:16px; margin:0; }
+.hero-stat-grid { display:grid; grid-template-columns:1fr 1fr; border:1px solid var(--line); border-radius:16px; overflow:hidden; background:rgba(255,255,255,.025); }
+.hero-stat { padding:17px 18px; min-height:78px; border-right:1px solid var(--line); border-bottom:1px solid var(--line); }
+.hero-stat:nth-child(2n){border-right:0}.hero-stat:nth-child(n+3){border-bottom:0}
+.hero-stat span { display:block; color:var(--muted); font-size:11px; margin-bottom:8px; }
+.hero-stat strong { font-size:15px; }
+.notice { border:1px solid rgba(245,199,110,.27); background:rgba(245,199,110,.07); color:#d9cfb8; border-radius:12px; padding:13px 16px; font-size:13px; line-height:1.5; }
+.tabs { display:flex; gap:8px; padding:26px 0 16px; overflow:auto; }
+.tab { border:0; color:var(--muted); background:transparent; padding:10px 14px; cursor:pointer; border-radius:8px; font-weight:650; }
+.tab:hover { color:var(--text); background:rgba(255,255,255,.035); }
+.tab.active { color:var(--text); background:rgba(139,124,255,.12); }
+.tab-panel { display:none; }.tab-panel.active { display:block; }
+.workspace { display:grid; grid-template-columns:400px minmax(0,1fr); gap:16px; align-items:start; }
+.panel { background:linear-gradient(180deg,rgba(19,25,37,.96),rgba(14,19,29,.96)); border:1px solid var(--line); border-radius:16px; box-shadow:var(--shadow); }
+.controls-panel { padding:20px; position:sticky; top:94px; }
+.result-panel,.wide-panel { padding:22px; }
+.panel-title-row { display:flex; align-items:flex-start; justify-content:space-between; gap:14px; margin-bottom:18px; }
+.panel-title-row h2 { margin:0; font-size:18px; letter-spacing:-.02em; }
+.panel-title-row p { margin:6px 0 0; }
+.tag { display:inline-flex; align-items:center; border:1px solid var(--line); background:rgba(255,255,255,.03); color:var(--muted); border-radius:99px; padding:5px 8px; font-size:10px; white-space:nowrap; }
+.tag.good { color:var(--good); border-color:rgba(99,217,165,.3); background:rgba(99,217,165,.07); }.tag.bad { color:var(--danger); border-color:rgba(255,123,138,.28); }.tag.neutral{color:var(--muted)}
+label { display:block; color:#bdc5d2; font-size:11px; font-weight:650; position:relative; }
+select,input { width:100%; margin-top:7px; height:38px; border:1px solid var(--line); background:#0b1018; color:var(--text); border-radius:8px; padding:0 10px; outline:none; }
+select:focus,input:focus { border-color:#625bd0; box-shadow:0 0 0 3px rgba(98,91,208,.12); }
+.unit { position:absolute; right:9px; bottom:11px; color:#697488; font-size:9px; pointer-events:none; }
+.field-grid { display:grid; gap:11px; margin-bottom:11px; }.field-grid.two { grid-template-columns:1fr 1fr; }
+hr { border:0; border-top:1px solid var(--line); margin:18px 0; }
+.section-kicker { margin-bottom:11px; }
+button.primary,button.secondary { width:100%; border-radius:9px; height:42px; border:1px solid transparent; margin-top:12px; font-weight:750; cursor:pointer; transition:.16s ease; }
+button.primary { background:linear-gradient(135deg,#7568f5,#4775f2); color:white; box-shadow:0 10px 26px rgba(94,85,222,.20); }
+button.primary:hover:not(:disabled){transform:translateY(-1px);filter:brightness(1.06)}
+button.secondary { background:transparent; color:#c7cfdb; border-color:var(--line); }
+button:disabled { opacity:.45; cursor:not-allowed; }.compact { width:auto !important; min-width:180px; padding:0 18px; margin-top:0 !important; }
+.empty-state { min-height:420px; display:grid; place-content:center; text-align:center; color:var(--muted); padding:30px; }.empty-state.small { min-height:260px; }.empty-state h3{color:#cfd6e2;margin:10px 0 6px;font-size:16px}.empty-state p{max-width:480px;margin:0;line-height:1.55;font-size:13px}.empty-icon{width:44px;height:44px;border:1px solid var(--line);border-radius:12px;display:grid;place-items:center;margin:0 auto;font-size:20px;color:var(--accent2)}
+.hidden { display:none !important; }
+.metric-grid { display:grid; grid-template-columns:repeat(6,1fr); gap:10px; margin-bottom:16px; }.metric-grid.four{grid-template-columns:repeat(4,1fr)}
+.metric { min-height:88px; padding:14px; border:1px solid var(--line); border-radius:11px; background:#0c111a; }.metric span{display:block;color:var(--muted);font-size:10px;margin-bottom:10px}.metric strong{font-size:19px;letter-spacing:-.03em}.metric.emphasis{border-color:rgba(139,124,255,.38);background:rgba(139,124,255,.06)}
+.chart-grid { display:grid; grid-template-columns:1fr 1fr; gap:12px; }.chart-card { border:1px solid var(--line); border-radius:12px; background:#0c111a; padding:14px; min-height:310px; }.chart-card.full{margin-top:12px}.chart-title{color:#cfd6e2;font-size:11px;font-weight:700;margin-bottom:12px}
+canvas { max-height:280px; }
+.warnings { margin-top:12px; border:1px solid rgba(245,199,110,.24); padding:11px 13px; background:rgba(245,199,110,.06); border-radius:9px; color:#d8cdaF; font-size:12px; }
+.muted { color:var(--muted); font-size:13px; line-height:1.55; }
+.table-wrap { overflow:auto; margin-top:14px; border:1px solid var(--line); border-radius:10px; }
+table { width:100%; border-collapse:collapse; font-size:12px; min-width:760px; } th,td{padding:11px 12px;text-align:right;border-bottom:1px solid var(--line)}th:first-child,td:first-child{text-align:left}th{color:#7f8ba0;font-size:10px;text-transform:uppercase;letter-spacing:.06em;background:#0a0f17}td{color:#cbd3df}tbody tr:last-child td{border-bottom:0}.pass{color:var(--good)}.fail{color:var(--danger)}
+.planner-grid { grid-template-columns:370px minmax(0,1fr); }
+.method-grid { display:grid; grid-template-columns:1fr 1fr; gap:16px; }.prose{padding:26px}.prose h2{font-size:22px;margin:8px 0 12px}.prose p{color:var(--muted);line-height:1.7;font-size:14px}.prose code{color:#b7b0ff}.formula{font-family:ui-monospace,SFMono-Regular,Menlo,monospace;color:#b9c6dc;padding:12px;border:1px solid var(--line);border-radius:8px;background:#0b1018;font-size:12px}.wide-method{grid-column:1/-1}.paper-grid{display:grid;grid-template-columns:repeat(5,1fr);gap:10px;margin-top:20px}.paper-grid>div{padding:14px;border:1px solid var(--line);border-radius:10px;background:#0c111a}.paper-grid strong{display:block;font-size:12px;margin-bottom:8px}.paper-grid span{color:var(--muted);font-size:11px;line-height:1.55;display:block}
+footer { width:min(1480px,calc(100% - 44px)); margin:0 auto; padding:22px 0 36px; border-top:1px solid rgba(255,255,255,.055); display:flex; justify-content:space-between; gap:20px; color:#657085; font-size:11px; }
+@media (max-width:1100px){.hero{grid-template-columns:1fr}.workspace,.planner-grid{grid-template-columns:1fr}.controls-panel{position:static}.metric-grid{grid-template-columns:repeat(3,1fr)}.paper-grid{grid-template-columns:1fr 1fr}}
+@media (max-width:720px){.topbar{padding:0 18px}.subtitle{display:none}.runtime-pill{max-width:170px}.shell{width:min(100% - 24px,1480px);padding-top:20px}.hero{padding-top:8px;gap:20px}.hero-stat-grid{grid-template-columns:1fr 1fr}.field-grid.two,.chart-grid,.method-grid{grid-template-columns:1fr}.metric-grid,.metric-grid.four{grid-template-columns:1fr 1fr}.paper-grid{grid-template-columns:1fr}.panel-title-row{flex-direction:column}.compact{width:100%!important}.wide-method{grid-column:auto}footer{width:calc(100% - 24px);flex-direction:column}}

tests/test_latency.py

@@ -0,0 +1,13 @@
+from inferscale.latency import AnalyticalLatencyModel
+from inferscale.profiles import get_accelerator, get_model
+
+
+def test_prefill_grows_with_tokens():
+    lm = AnalyticalLatencyModel(get_model("Llama-3.1-8B"), get_accelerator("L4"))
+    assert lm.prefill_seconds([1024]) > lm.prefill_seconds([128])
+
+
+def test_quantization_reduces_weight_memory():
+    fp16 = AnalyticalLatencyModel(get_model("Llama-3.1-8B"), get_accelerator("L4"), "fp16")
+    int4 = AnalyticalLatencyModel(get_model("Llama-3.1-8B"), get_accelerator("L4"), "int4")
+    assert int4.model_weight_gb < fp16.model_weight_gb

tests/test_optimizer.py

@@ -0,0 +1,20 @@
+from inferscale.optimizer import capacity_search, compare_schedulers
+
+
+def test_capacity_search_returns_trace():
+    cfg = {
+        "duration_s": 6,
+        "prompt_tokens_mean": 128,
+        "output_tokens_mean": 16,
+        "slo_ttft_ms": 1000,
+        "slo_e2e_ms": 10000,
+        "slo_attainment_target": 0.95,
+    }
+    out = capacity_search(cfg, min_rate=0.25, max_rate=3, iterations=3, repetitions=1)
+    assert out["trace"]
+    assert out["capacity_rps"] >= 0
+
+
+def test_scheduler_compare_has_rows():
+    out = compare_schedulers({"duration_s": 4, "request_rate_rps": 1, "output_tokens_mean": 8})
+    assert len(out["rows"]) == 5

tests/test_simulator.py

@@ -0,0 +1,42 @@
+from inferscale import run_simulation
+
+
+def base_config():
+    return {
+        "duration_s": 8,
+        "request_rate_rps": 1.5,
+        "prompt_tokens_mean": 128,
+        "prompt_tokens_cv": 0.1,
+        "output_tokens_mean": 16,
+        "output_tokens_cv": 0.1,
+        "seed": 3,
+    }
+
+
+def test_simulation_completes_requests():
+    result = run_simulation(base_config())
+    assert result["summary"]["requests_generated"] > 0
+    assert result["summary"]["requests_unfinished"] == 0
+    assert result["latency"]["ttft_ms"]["p95"] > 0
+    assert result["resource"]["peak_kv_gb"] >= 0
+    assert result["provenance"]["latency_profile_type"] == "analytical-reference"
+
+
+def test_high_load_increases_tail_latency():
+    low = base_config()
+    high = base_config()
+    low["request_rate_rps"] = 0.5
+    high["request_rate_rps"] = 8.0
+    a = run_simulation(low)
+    b = run_simulation(high)
+    assert b["latency"]["ttft_ms"]["p95"] >= a["latency"]["ttft_ms"]["p95"]
+
+
+def test_static_and_continuous_are_distinct():
+    cfg = base_config()
+    cfg.update({"request_rate_rps": 4.0, "output_tokens_mean": 32})
+    cfg["scheduler"] = "static_fcfs"
+    static = run_simulation(cfg)
+    cfg["scheduler"] = "continuous_fcfs"
+    continuous = run_simulation(cfg)
+    assert static["summary"]["goodput_rps"] != continuous["summary"]["goodput_rps"] or static["latency"]["ttft_ms"]["p95"] != continuous["latency"]["ttft_ms"]["p95"]

tests/test_workloads.py

@@ -0,0 +1,17 @@
+from inferscale.models import SimulationConfig
+from inferscale.workloads import generate_workload
+
+
+def test_workload_is_deterministic():
+    cfg = SimulationConfig(seed=42, duration_s=10, request_rate_rps=2)
+    a = generate_workload(cfg)
+    b = generate_workload(cfg)
+    assert [(r.arrival_time, r.prompt_tokens, r.output_tokens) for r in a] == [
+        (r.arrival_time, r.prompt_tokens, r.output_tokens) for r in b
+    ]
+
+
+def test_constant_arrivals():
+    cfg = SimulationConfig(arrival_process="constant", duration_s=2, request_rate_rps=2)
+    reqs = generate_workload(cfg)
+    assert [round(r.arrival_time, 3) for r in reqs] == [0.0, 0.5, 1.0, 1.5]

worker.mjs

@@ -0,0 +1,56 @@
+import { loadPyodide } from "https://cdn.jsdelivr.net/pyodide/v314.0.5/full/pyodide.mjs";
+
+const MODULES = [
+  "__init__.py",
+  "api.py",
+  "kv_cache.py",
+  "latency.py",
+  "metrics.py",
+  "models.py",
+  "optimizer.py",
+  "profiles.py",
+  "simulator.py",
+  "workloads.py",
+];
+
+let pyodide;
+
+async function init() {
+  pyodide = await loadPyodide();
+  pyodide.FS.mkdirTree("/home/pyodide/inferscale");
+  for (const file of MODULES) {
+    const url = new URL(`./py/inferscale/${file}`, self.location.href);
+    const response = await fetch(url);
+    if (!response.ok) throw new Error(`Failed to fetch ${file}: HTTP ${response.status}`);
+    pyodide.FS.writeFile(`/home/pyodide/inferscale/${file}`, await response.text(), { encoding: "utf8" });
+  }
+  await pyodide.runPythonAsync(`
+import sys
+if "/home/pyodide" not in sys.path:
+    sys.path.insert(0, "/home/pyodide")
+from inferscale.api import execute
+`);
+  self.postMessage({ type: "ready" });
+}
+
+const readyPromise = init().catch((error) => {
+  self.postMessage({ type: "fatal", error: error.message || String(error) });
+  throw error;
+});
+
+self.onmessage = async (event) => {
+  const { id, action, payload } = event.data;
+  try {
+    await readyPromise;
+    pyodide.globals.set("_action", action);
+    pyodide.globals.set("_payload_json", JSON.stringify(payload ?? {}));
+    const output = await pyodide.runPythonAsync(`
+import json
+_result = execute(_action, json.loads(_payload_json))
+json.dumps(_result, separators=(",", ":"))
+`);
+    self.postMessage({ id, result: JSON.parse(output) });
+  } catch (error) {
+    self.postMessage({ id, error: error.message || String(error) });
+  }
+};