Spaces:

Exeaon
/

memory-wall

Running

File size: 9,164 Bytes

<!doctype html>
<html lang="en">
<head>
<meta charset="utf-8">
<meta name="viewport" content="width=device-width,initial-scale=1">
<title>The Memory Wall — Exeaon</title>
<style>
:root{
  --bg:#fbfbfd; --panel:#fff; --ink:#14151a; --muted:#6b6f7b;
  --line:#e6e7ec; --accent:#4f46e5; --good:#0f9d58; --bad:#d93025;
  --mono:ui-monospace,SFMono-Regular,"SF Mono",Menlo,Consolas,monospace;
}
@media (prefers-color-scheme:dark){
  :root{--bg:#0d0e12;--panel:#15171d;--ink:#e9eaee;--muted:#9096a3;
        --line:#252832;--accent:#8b85ff;--good:#4ade80;--bad:#f87171;}
}
*{box-sizing:border-box}
body{margin:0;padding:2rem 1.25rem 4rem;background:var(--bg);color:var(--ink);
  font:16px/1.6 -apple-system,BlinkMacSystemFont,"Segoe UI",Inter,sans-serif;}
.wrap{max-width:880px;margin:0 auto}
h1{font-size:2.1rem;letter-spacing:-.03em;margin:0 0 .4rem}
.sub{color:var(--muted);margin:0 0 2rem}
.eq{background:var(--panel);border:1px solid var(--line);border-left:3px solid var(--accent);
  border-radius:8px;padding:1rem 1.25rem;margin:0 0 2rem;font-family:var(--mono);
  font-size:1.05rem;overflow-x:auto;white-space:nowrap}
.controls{display:grid;gap:1.25rem;grid-template-columns:repeat(auto-fit,minmax(230px,1fr));
  background:var(--panel);border:1px solid var(--line);border-radius:10px;padding:1.25rem;margin-bottom:1.5rem}
label{display:block;font-size:.82rem;font-weight:600;letter-spacing:.02em;
  text-transform:uppercase;color:var(--muted);margin-bottom:.5rem}
input[type=range]{width:100%;accent-color:var(--accent)}
select{width:100%;padding:.55rem;border-radius:6px;border:1px solid var(--line);
  background:var(--bg);color:var(--ink);font-size:.95rem}
.val{font-family:var(--mono);font-size:1.15rem;font-weight:600}
.verdict{background:var(--panel);border:1px solid var(--line);border-radius:10px;
  padding:1.25rem 1.4rem;margin-bottom:1.5rem}
.verdict h2{margin:0 0 .5rem;font-size:1.35rem;letter-spacing:-.02em}
.verdict p{margin:0;color:var(--muted)}
.tblwrap{overflow-x:auto;border:1px solid var(--line);border-radius:10px;background:var(--panel)}
table{border-collapse:collapse;width:100%;font-size:.92rem;min-width:560px}
th,td{padding:.7rem .9rem;text-align:right;border-bottom:1px solid var(--line);white-space:nowrap}
th:first-child,td:first-child{text-align:left}
th{font-size:.75rem;text-transform:uppercase;letter-spacing:.04em;color:var(--muted);font-weight:600}
tbody tr:last-child td{border-bottom:none}
tr.ours{background:color-mix(in srgb,var(--accent) 7%,transparent)}
td.num{font-family:var(--mono)}
.yes{color:var(--good);font-weight:600}
.no{color:var(--bad);font-weight:600}
.note{color:var(--muted);font-size:.88rem;margin-top:1.75rem;padding-top:1.25rem;border-top:1px solid var(--line)}
a{color:var(--accent)}
</style>
</head>
<body>
<div class="wrap">

<h1>The Memory Wall</h1>
<p class="sub">Why your GPU's TFLOPs do not decide your tokens per second.</p>

<div class="eq">tokens/second &nbsp;&le;&nbsp; memory bandwidth &nbsp;&divide;&nbsp; model bytes</div>

<p>At batch one, a language model reads <strong>every weight from memory to produce
one token</strong>, and uses each weight exactly once. Arithmetic intensity is about
<strong>0.5 FLOP per byte</strong>. The accelerator's FLOP rating never enters the
equation — the only lever on single-stream speed is how many bytes the model is.</p>

<div class="controls">
  <div>
    <label for="p">Parameters <span class="val" id="pv">7B</span></label>
    <input type="range" id="p" min="0.5" max="120" step="0.1" value="7">
  </div>
  <div>
    <label for="hw">Hardware</label>
    <select id="hw"></select>
  </div>
  <div>
    <label for="m">Available memory <span class="val" id="mv">16 GB</span></label>
    <input type="range" id="m" min="4" max="192" step="1" value="16">
  </div>
</div>

<div class="verdict"><h2 id="vh"></h2><p id="vp"></p></div>

<div class="tblwrap"><table>
<thead><tr><th>Format</th><th>Bits/weight</th><th>Size</th><th>Fits</th><th>Decode ceiling</th></tr></thead>
<tbody id="rows"></tbody>
</table></div>

<p class="note">
<strong>This is a ceiling, not a promise.</strong> Attention, the KV cache, kernel
overhead and real bandwidth utilisation (typically 70–85% of the spec sheet) all
put the achievable number below it. Larger batches raise arithmetic intensity and
change the picture entirely — this models the single-stream case, which is what
most people run locally. Bandwidth figures are vendor peak.
</p>
<h2 style="margin-top:34px">Models built against this limit</h2>
<p class="note" style="border:none;padding-top:0">
Bytes are the only lever at batch one, so these are what the lever is worth in
practice. Accuracy retained is the mean of ARC-Easy, ARC-Challenge, HellaSwag and
PIQA against the uncompressed model, scored on the same GPU and harness. The
speech model has no such benchmark, so it is measured by word error rate instead.
</p>

<div class="tblwrap"><table>
<thead><tr><th>Model</th><th>Base</th><th>Size</th><th>Smaller by</th><th>Accuracy retained</th></tr></thead>
<tbody>
<tr><td><a href="https://huggingface.co/Exeaon/Exeaon1-Nunya-14B">Exeaon1-Nunya-14B</a></td><td>Qwen3-14B</td><td>7.38 GB</td><td>3.73x</td><td>100.3%</td></tr>
<tr><td><a href="https://huggingface.co/Exeaon/Exeaon1-Kese-30B-A3B">Exeaon1-Kese-30B-A3B</a></td><td>Qwen3-30B-A3B</td><td>16.27 GB</td><td>3.50x</td><td>98.1%</td></tr>
<tr><td><a href="https://huggingface.co/Exeaon/Exeaon1-Nunya-8B">Exeaon1-Nunya-8B</a></td><td>Qwen3-8B</td><td>4.11 GB</td><td>3.80x</td><td>99.6%</td></tr>
<tr><td><a href="https://huggingface.co/Exeaon/Exeaon1-Dzo-4B">Exeaon1-Dzo-4B</a></td><td>Qwen3-4B</td><td>2.04 GB</td><td>3.67x</td><td>98.1%</td></tr>
<tr><td><a href="https://huggingface.co/Exeaon/Exeaon1-Dzo-0.6B">Exeaon1-Dzo-0.6B</a></td><td>Qwen3-0.6B</td><td>0.31 GB</td><td>3.50x</td><td>96.3%</td></tr>
<tr><td><a href="https://huggingface.co/Exeaon/Exeaon1-Voice-0.8B">Exeaon1-Voice-0.8B</a></td><td>Whisper v3-turbo</td><td>0.43 GB</td><td>3.47x</td><td>WER unchanged</td></tr>
</tbody>
</table></div>

<p class="note">
<strong>Retention is not a free lunch, and neither is speed.</strong> On a large
GPU these run slower than dense fp16 — a vendor tensor-core GEMM is heavily
tuned and there is bandwidth to spare, so trading compute for memory loses. The
win is on the other side of this page: fitting where the dense model does not.
</p>

<p class="note" style="border:none;padding-top:0">
Compressed models: <a href="https://huggingface.co/Exeaon">huggingface.co/Exeaon</a> ·
Runtime: <a href="https://github.com/ExeaonLM">github.com/ExeaonLM</a> ·
Zenux Plimver Technologies LTD, Ghana
</p>

</div>
<script>
const HW = {
  "Laptop CPU (DDR5-5600, dual channel)":90,
  "Desktop CPU (DDR5-6000, dual channel)":96,
  "Server CPU (DDR5, 12 channel)":460,
  "Apple M4 Pro":273, "Apple M4 Max":546,
  "NVIDIA T4":320, "NVIDIA RTX 4090":1008, "NVIDIA L40S":864,
  "NVIDIA A100 80GB":2039, "NVIDIA H100 SXM":3350, "NVIDIA B200":8000,
};
const FMT = [
  ["FP16 / BF16",16,false],["FP8",8,false],["INT8",8,false],["INT4",4,false],
  ["ℰ-PURE 4-bit",4,true],["ℰ-PURE 3-bit",3,true],
];
const $=id=>document.getElementById(id);
const sel=$("hw");
for(const k in HW){const o=document.createElement("option");o.textContent=k;sel.appendChild(o);}
sel.value="NVIDIA T4";

// Weight bytes only. KV cache and activations are extra, which is part of why
// the printed number is a ceiling rather than an estimate.
const gb=(p,bits)=>p*1e9*bits/8/2**30;

function render(){
  const p=+$("p").value, mem=+$("m").value, bw=HW[sel.value];
  $("pv").textContent=p.toFixed(1).replace(/\.0$/,"")+"B";
  $("mv").textContent=mem+" GB";

  const rows=FMT.map(([name,bits,ours])=>{
    const size=gb(p,bits);
    return {name,bits,ours,size,fits:size<=mem*0.9,tps:bw/size};
  });

  $("rows").innerHTML=rows.map(r=>`<tr class="${r.ours?"ours":""}">
    <td>${r.name}</td><td class="num">${r.bits}</td>
    <td class="num">${r.size.toFixed(2)} GB</td>
    <td class="${r.fits?"yes":"no"}">${r.fits?"yes":"no"}</td>
    <td class="num">${r.tps.toFixed(1)} tok/s</td></tr>`).join("");

  const base=rows[0], fitting=rows.filter(r=>r.fits), best=rows[rows.length-1];
  let h,t;
  if(!fitting.length){
    h=`Nothing fits in ${mem} GB`;
    t=`A ${p.toFixed(1).replace(/\.0$/,"")}B model exceeds this device's memory at every
       format listed. Smaller model, or more memory.`;
  }else if(!base.fits){
    const f=fitting[fitting.length-1];
    h=`${sel.value} cannot run this model in FP16 at all`;
    t=`FP16 needs ${base.size.toFixed(1)} GB against ${mem} GB available. At
       ${f.name} it is ${f.size.toFixed(1)} GB and fits — a step function, not a
       speed-up. The model goes from impossible to running at roughly
       ${f.tps.toFixed(0)} tok/s.`;
  }else{
    h=`${best.tps.toFixed(0)} vs ${base.tps.toFixed(0)} tok/s`;
    t=`Same model, same hardware, same ${bw} GB/s of bandwidth.
       ${(best.tps/base.tps).toFixed(1)}× more tokens purely from moving fewer bytes.`;
  }
  $("vh").textContent=h; $("vp").textContent=t;
}
["p","m","hw"].forEach(id=>$(id).addEventListener("input",render));
render();
</script>
</body>
</html>