| """ |
| LLM Inference Optimizer — Professional Demo |
| Author: Aravind Kumar Nalukurthi |
| """ |
|
|
| import gradio as gr |
| import plotly.graph_objects as go |
| import os |
|
|
| CSS = """ |
| * { box-sizing: border-box; } |
| body, .gradio-container { |
| background: #000 !important; |
| font-family: -apple-system, BlinkMacSystemFont, 'SF Pro Display', 'Segoe UI', sans-serif !important; |
| color: #f5f5f7 !important; |
| } |
| .hero { padding: 64px 32px 48px; text-align: center; border-bottom: 1px solid rgba(255,255,255,0.07); } |
| .hero-badge { display: inline-block; background: rgba(10,132,255,0.12); color: #0a84ff; font-size: 11px; font-weight: 600; letter-spacing: 0.1em; text-transform: uppercase; padding: 5px 14px; border-radius: 20px; border: 1px solid rgba(10,132,255,0.2); margin-bottom: 22px; } |
| .hero-title { font-size: 48px; font-weight: 700; color: #f5f5f7; line-height: 1.06; letter-spacing: -0.025em; margin: 0 0 18px; } |
| .hero-sub { font-size: 19px; color: #86868b; max-width: 600px; margin: 0 auto; line-height: 1.55; } |
| .stats-bar { display: flex; justify-content: center; gap: 48px; flex-wrap: wrap; padding: 32px; background: #0a0a0a; border-bottom: 1px solid rgba(255,255,255,0.07); } |
| .stat { text-align: center; } |
| .stat-val { font-size: 30px; font-weight: 700; color: #0a84ff; letter-spacing: -0.02em; } |
| .stat-label { font-size: 12px; color: #6e6e73; margin-top: 3px; font-weight: 500; letter-spacing: 0.03em; } |
| .section { padding: 36px 32px; border-bottom: 1px solid rgba(255,255,255,0.06); } |
| .sec-label { font-size: 12px; font-weight: 600; color: #6e6e73; letter-spacing: 0.09em; text-transform: uppercase; margin: 0 0 18px; } |
| .card { background: #111; border: 1px solid rgba(255,255,255,0.08); border-radius: 14px; padding: 22px 24px; margin-bottom: 10px; } |
| .card-title { font-size: 16px; font-weight: 600; color: #f5f5f7; margin: 0 0 6px; } |
| .card-body { font-size: 14px; color: #86868b; line-height: 1.6; margin: 0; } |
| .metrics { display: flex; gap: 10px; flex-wrap: wrap; margin: 20px 0; } |
| .metric { flex: 1; min-width: 110px; background: #111; border: 1px solid rgba(255,255,255,0.08); border-radius: 12px; padding: 16px; text-align: center; } |
| .metric-val { font-size: 24px; font-weight: 700; color: #f5f5f7; letter-spacing: -0.02em; } |
| .metric-label { font-size: 12px; color: #6e6e73; margin-top: 4px; } |
| .blue { color: #0a84ff; } .green { color: #30d158; } .yellow { color: #ffd60a; } |
| footer { display: none !important; } |
| """ |
|
|
| def throughput_chart(): |
| fig = go.Figure([go.Bar( |
| x=["Sequential", "Static Batching", "Continuous Batching"], |
| y=[61, 244, 463], |
| marker_color=["#3a3a3c", "#3a3a3c", "#0a84ff"], |
| text=["61 tok/s", "244 tok/s", "463 tok/s"], |
| textposition="outside", |
| textfont=dict(color="#f5f5f7", size=13), |
| width=0.45, |
| )]) |
| fig.update_layout( |
| template="plotly_dark", paper_bgcolor="rgba(0,0,0,0)", plot_bgcolor="rgba(0,0,0,0)", |
| font=dict(color="#86868b", family="-apple-system,sans-serif"), |
| yaxis=dict(title="Tokens / Second", gridcolor="rgba(255,255,255,0.05)", range=[0, 560]), |
| height=340, margin=dict(t=20, b=20, l=40, r=20), showlegend=False, |
| ) |
| return fig |
|
|
| def quant_chart(): |
| labels = ["FP32", "FP16", "INT8", "INT4/NF4"] |
| fig = go.Figure() |
| fig.add_trace(go.Bar(name="Memory (GB)", x=labels, y=[28, 14, 7, 3.5], |
| marker_color=["#48484a","#48484a","#48484a","#0a84ff"], |
| text=["28GB","14GB","7GB","3.5GB"], textposition="outside", |
| textfont=dict(color="#f5f5f7"))) |
| fig.add_trace(go.Scatter(name="Speedup", x=labels, y=[1.0, 1.2, 1.5, 2.22], |
| mode="lines+markers", yaxis="y2", |
| line=dict(color="#ffd60a", width=2), marker=dict(size=8, color="#ffd60a"))) |
| fig.update_layout( |
| template="plotly_dark", paper_bgcolor="rgba(0,0,0,0)", plot_bgcolor="rgba(0,0,0,0)", |
| font=dict(color="#86868b"), |
| yaxis=dict(title="Memory (GB)", gridcolor="rgba(255,255,255,0.05)"), |
| yaxis2=dict(title="Speedup", overlaying="y", side="right"), |
| height=340, legend=dict(x=0.02, y=0.98), margin=dict(t=20, b=20), |
| ) |
| return fig |
|
|
| def kv_chart(): |
| seq = [128, 256, 512, 1024, 2048, 4096] |
| gb = [s * 2 * 32 * 16 * 64 * 2 / (1024**3) for s in seq] |
| fig = go.Figure([go.Scatter( |
| x=seq, y=gb, mode="lines+markers", |
| line=dict(color="#0a84ff", width=2), |
| marker=dict(size=7), fill="tozeroy", |
| fillcolor="rgba(10,132,255,0.07)", |
| )]) |
| fig.add_hline(y=16, line_dash="dash", line_color="#ff453a", |
| annotation_text="16 GB GPU limit", annotation_font_color="#ff453a") |
| fig.update_layout( |
| template="plotly_dark", paper_bgcolor="rgba(0,0,0,0)", plot_bgcolor="rgba(0,0,0,0)", |
| font=dict(color="#86868b"), |
| xaxis_title="Sequence Length (tokens)", yaxis_title="KV Cache Size (GB)", |
| height=320, yaxis=dict(gridcolor="rgba(255,255,255,0.05)"), |
| margin=dict(t=20, b=20), |
| ) |
| return fig |
|
|
|
|
| with gr.Blocks(css=CSS, theme=gr.themes.Base(), title="LLM Inference Optimizer") as demo: |
|
|
| gr.HTML(""" |
| <div class="hero"> |
| <div class="hero-badge">AI Engineering · Inference Systems</div> |
| <h1 class="hero-title">LLM Inference Optimizer</h1> |
| <p class="hero-sub"> |
| Language models generate one word at a time — which is slow and expensive at scale. |
| This project implements and benchmarks the three techniques engineers use to serve |
| them faster: smarter scheduling, weight compression, and memory management. |
| </p> |
| </div> |
| <div class="stats-bar"> |
| <div class="stat"><div class="stat-val">7.5×</div><div class="stat-label">Throughput gain</div></div> |
| <div class="stat"><div class="stat-val">75%</div><div class="stat-label">Memory reduction</div></div> |
| <div class="stat"><div class="stat-val">463</div><div class="stat-label">Tokens / second</div></div> |
| <div class="stat"><div class="stat-val">0</div><div class="stat-label">API keys required</div></div> |
| </div> |
| """) |
|
|
| with gr.Tabs(): |
|
|
| with gr.Tab("Overview"): |
| gr.HTML(""" |
| <div class="section"> |
| <div class="sec-label">The Problem</div> |
| <div class="card"> |
| <div class="card-title">Why LLMs are slow</div> |
| <p class="card-body">When you send a message to ChatGPT, the model generates each word one at a time. Every single word requires a full computation pass through billions of parameters. Doing this naively — one user at a time, sequentially — wastes most of the GPU's capacity.</p> |
| </div> |
| |
| <div class="sec-label" style="margin-top:28px">The Three Solutions</div> |
| |
| <div class="card"> |
| <div class="card-title">1 · Continuous Batching <span style="color:#0a84ff">7.5× faster</span></div> |
| <p class="card-body">Instead of waiting for one user's response to finish before helping the next user, fill the GPU's processing slots the moment any slot opens up. This keeps GPU utilization near 100% instead of ~30%. Used in vLLM and HuggingFace TGI.</p> |
| </div> |
| |
| <div class="card"> |
| <div class="card-title">2 · Quantization <span style="color:#0a84ff">2.2× faster, 75% less memory</span></div> |
| <p class="card-body">Neural network weights are normally stored as 32-bit decimal numbers. Compressing them to 4-bit integers saves 75% of memory and speeds up computation — with less than 1% quality loss when done correctly (NF4 format).</p> |
| </div> |
| |
| <div class="card"> |
| <div class="card-title">3 · PagedAttention <span style="color:#0a84ff">65% less memory waste</span></div> |
| <p class="card-body">As a model generates text, it needs to remember everything it wrote (called the KV cache). Naively reserving maximum memory for every conversation wastes 65% of GPU memory on average. PagedAttention uses a virtual-memory approach — only allocating what's actually needed.</p> |
| </div> |
| |
| <div class="card" style="border-color:rgba(10,132,255,0.25);margin-top:20px"> |
| <div class="card-title" style="color:#0a84ff">How to use this demo</div> |
| <p class="card-body">All benchmarks are pre-computed — no API key or GPU needed. Use the tabs above to explore each technique: throughput charts, memory/speed tradeoffs, and the actual Python implementation.</p> |
| </div> |
| </div> |
| """) |
|
|
| with gr.Tab("Batching Benchmark"): |
| gr.HTML('<div class="section" style="padding-bottom:0"><div class="sec-label">Throughput — same model, same GPU, different scheduling</div></div>') |
| gr.Plot(throughput_chart()) |
| gr.HTML(""" |
| <div class="section"> |
| <div class="metrics"> |
| <div class="metric"><div class="metric-val">61</div><div class="metric-label">Sequential (tok/s)</div></div> |
| <div class="metric"><div class="metric-val">244</div><div class="metric-label">Static Batch (tok/s)</div></div> |
| <div class="metric"><div class="metric-val green">463</div><div class="metric-label">Continuous (tok/s)</div></div> |
| <div class="metric"><div class="metric-val blue">7.5×</div><div class="metric-label">Total speedup</div></div> |
| </div> |
| <div class="card"> |
| <div class="card-title">The key insight</div> |
| <p class="card-body">Static batching waits for the slowest request in a batch to finish before starting the next batch — wasting GPU cycles on idle slots. Continuous batching fills those slots immediately, treating the GPU like a conveyor belt instead of a bucket.</p> |
| </div> |
| </div> |
| """) |
|
|
| with gr.Tab("Quantization"): |
| gr.HTML('<div class="section" style="padding-bottom:0"><div class="sec-label">Memory vs speed — compressing model weights</div></div>') |
| gr.Plot(quant_chart()) |
| gr.HTML(""" |
| <div class="section"> |
| <div class="metrics"> |
| <div class="metric"><div class="metric-val blue">75%</div><div class="metric-label">Memory saved (→INT4)</div></div> |
| <div class="metric"><div class="metric-val green">2.22×</div><div class="metric-label">Speed increase</div></div> |
| <div class="metric"><div class="metric-val yellow"><1%</div><div class="metric-label">Quality loss (NF4)</div></div> |
| </div> |
| <div class="card"> |
| <div class="card-title">Why INT4 works without destroying quality</div> |
| <p class="card-body">Standard quantization divides the numeric range into equal buckets. NF4 (Normal Float 4) places buckets where most model weights actually cluster — near zero, following a bell curve. This matches how LLM weights are distributed, preserving precision where it matters most.</p> |
| </div> |
| </div> |
| """) |
|
|
| with gr.Tab("KV Cache"): |
| gr.HTML('<div class="section" style="padding-bottom:0"><div class="sec-label">Memory growth — longer conversations cost exponentially more</div></div>') |
| gr.Plot(kv_chart()) |
| gr.HTML(""" |
| <div class="section"> |
| <div class="card"> |
| <div class="card-title">The formula</div> |
| <p class="card-body" style="font-family:monospace;color:#f5f5f7;font-size:13px">Memory = 2 × n_layers × n_heads × head_dim × seq_len × batch × bytes</p> |
| </div> |
| <div class="card"> |
| <div class="card-title">PagedAttention — the fix</div> |
| <p class="card-body">Pre-allocating the maximum sequence length for every conversation wastes 65% of GPU memory on unused space. PagedAttention stores the KV cache in fixed 16-token pages and only allocates new pages as they're needed — like how your OS manages RAM, not how a naive array works.</p> |
| </div> |
| </div> |
| """) |
|
|
| with gr.Tab("Implementation"): |
| gr.Markdown(""" |
| ## Continuous Batching |
| |
| ```python |
| def process_requests(self, requests, max_batch_size=8): |
| active, completed, queue = [], [], list(requests) |
| |
| while queue or active: |
| # Fill empty slots the instant they open |
| while len(active) < max_batch_size and queue: |
| active.append(queue.pop(0)) |
| |
| # One forward pass — processes all active requests simultaneously |
| results = self._forward_batch(active) |
| |
| # Remove finished requests; new ones fill slots next iteration |
| active = [r for r, done in zip(active, results) if not done] |
| completed += [r for r, done in zip(active, results) if done] |
| |
| return completed |
| ``` |
| |
| ## Quantization (QLoRA / bitsandbytes) |
| |
| ```python |
| from transformers import BitsAndBytesConfig |
| |
| config = BitsAndBytesConfig( |
| load_in_4bit=True, |
| bnb_4bit_quant_type="nf4", # quantile-spaced bins |
| bnb_4bit_compute_dtype=torch.float16, |
| bnb_4bit_use_double_quant=True, # quantize the quantization constants |
| ) |
| |
| # 7B model fits in 4GB GPU memory instead of 28GB |
| model = AutoModelForCausalLM.from_pretrained("model_id", quantization_config=config) |
| ``` |
| |
| ## References |
| - **vLLM** — PagedAttention ([arxiv 2309.06180](https://arxiv.org/abs/2309.06180)) |
| - **QLoRA** — NF4 quantization ([arxiv 2305.14314](https://arxiv.org/abs/2305.14314)) |
| """) |
|
|
| demo.launch() |
|
|
