hiaripc commited on
Commit ·
72ec23c
1
Parent(s): 708d56c
- index.html +131 -230
index.html
CHANGED
|
@@ -1,297 +1,198 @@
|
|
| 1 |
<!DOCTYPE html>
|
| 2 |
-
|
| 3 |
-
<html lang="en">
|
| 4 |
-
|
| 5 |
<head>
|
| 6 |
<meta charset="UTF-8">
|
| 7 |
-
<meta name="viewport" content="width=device-width, initial-scale=1.0">
|
| 8 |
<title>SweetSpot v2 — Energy-per-Token demo</title>
|
| 9 |
|
| 10 |
-
|
| 11 |
-
<script src="https://cdn.plot.ly/plotly-latest.min.js"></script>
|
| 12 |
-
|
| 13 |
-
<style>
|
| 14 |
-
* {
|
| 15 |
-
box-sizing: border-box;
|
| 16 |
-
}
|
| 17 |
-
|
| 18 |
-
body {
|
| 19 |
-
max-width: 1000px;
|
| 20 |
-
margin: 0 auto;
|
| 21 |
-
padding: 24px;
|
| 22 |
-
font-family: system-ui, -apple-system, BlinkMacSystemFont, "Segoe UI", sans-serif;
|
| 23 |
-
line-height: 1.5;
|
| 24 |
-
color: #222;
|
| 25 |
-
}
|
| 26 |
-
|
| 27 |
-
h2 {
|
| 28 |
-
margin-bottom: 8px;
|
| 29 |
-
}
|
| 30 |
-
|
| 31 |
-
h4 {
|
| 32 |
-
margin: 28px 0 10px;
|
| 33 |
-
}
|
| 34 |
-
|
| 35 |
-
p {
|
| 36 |
-
margin: 12px 0;
|
| 37 |
-
}
|
| 38 |
-
|
| 39 |
-
ul {
|
| 40 |
-
margin-top: 6px;
|
| 41 |
-
}
|
| 42 |
-
|
| 43 |
-
button {
|
| 44 |
-
padding: 8px 14px;
|
| 45 |
-
border: 1px solid #bbb;
|
| 46 |
-
border-radius: 6px;
|
| 47 |
-
background: #f5f5f5;
|
| 48 |
-
cursor: pointer;
|
| 49 |
-
}
|
| 50 |
-
|
| 51 |
-
button:hover {
|
| 52 |
-
background: #eaeaea;
|
| 53 |
-
}
|
| 54 |
-
|
| 55 |
-
select,
|
| 56 |
-
input[type="text"],
|
| 57 |
-
input[type="number"] {
|
| 58 |
-
padding: 7px 9px;
|
| 59 |
-
border: 1px solid #bbb;
|
| 60 |
-
border-radius: 5px;
|
| 61 |
-
}
|
| 62 |
-
|
| 63 |
-
#preset {
|
| 64 |
-
min-width: 280px;
|
| 65 |
-
}
|
| 66 |
-
|
| 67 |
-
#repo {
|
| 68 |
-
width: min(400px, 100%);
|
| 69 |
-
}
|
| 70 |
-
|
| 71 |
-
.architecture-grid {
|
| 72 |
-
display: grid;
|
| 73 |
-
grid-template-columns: repeat(4, 1fr);
|
| 74 |
-
gap: 16px;
|
| 75 |
-
}
|
| 76 |
|
| 77 |
-
|
| 78 |
-
|
| 79 |
-
|
| 80 |
-
|
| 81 |
-
|
| 82 |
-
|
| 83 |
-
|
| 84 |
-
}
|
| 85 |
-
|
| 86 |
-
/* Token slider */
|
| 87 |
-
|
| 88 |
-
#ninWrapper {
|
| 89 |
-
margin-top: 24px;
|
| 90 |
-
width: 100%;
|
| 91 |
-
}
|
| 92 |
-
|
| 93 |
-
.slider-header {
|
| 94 |
-
display: flex;
|
| 95 |
-
justify-content: space-between;
|
| 96 |
-
align-items: baseline;
|
| 97 |
-
margin-bottom: 8px;
|
| 98 |
-
}
|
| 99 |
-
|
| 100 |
-
.slider-value {
|
| 101 |
-
font-variant-numeric: tabular-nums;
|
| 102 |
-
font-weight: 600;
|
| 103 |
-
}
|
| 104 |
-
|
| 105 |
-
.slider-track {
|
| 106 |
-
width: 100%;
|
| 107 |
-
}
|
| 108 |
-
|
| 109 |
-
#ninSlider {
|
| 110 |
-
display: block;
|
| 111 |
-
width: 100%;
|
| 112 |
-
margin: 0;
|
| 113 |
-
}
|
| 114 |
-
|
| 115 |
-
#ninLabels {
|
| 116 |
-
display: flex;
|
| 117 |
-
justify-content: space-between;
|
| 118 |
-
margin-top: 5px;
|
| 119 |
-
padding: 0 7px;
|
| 120 |
-
font-size: 12px;
|
| 121 |
-
color: #666;
|
| 122 |
-
}
|
| 123 |
-
|
| 124 |
-
#ninLabels span {
|
| 125 |
-
text-align: center;
|
| 126 |
-
}
|
| 127 |
-
|
| 128 |
-
#plot {
|
| 129 |
-
width: 100%;
|
| 130 |
-
margin-top: 24px;
|
| 131 |
-
}
|
| 132 |
|
| 133 |
-
|
| 134 |
-
|
| 135 |
-
|
|
|
|
| 136 |
}
|
| 137 |
-
}
|
| 138 |
|
| 139 |
-
|
| 140 |
-
|
| 141 |
-
|
| 142 |
}
|
| 143 |
|
| 144 |
-
|
| 145 |
-
|
| 146 |
}
|
| 147 |
-
}
|
| 148 |
-
</style>
|
| 149 |
-
```
|
| 150 |
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
| 151 |
</head>
|
| 152 |
|
| 153 |
<body>
|
| 154 |
|
| 155 |
-
|
| 156 |
-
|
|
|
|
| 157 |
|
| 158 |
<p>
|
| 159 |
-
|
| 160 |
-
|
| 161 |
-
number of output tokens (<i>n</i><sub>out</sub>).
|
| 162 |
-
Select one of the validated models used in the paper or load the
|
| 163 |
-
architecture of any <b>dense decoder-only Transformer</b> (up to
|
| 164 |
-
approximately 20B parameters) from Hugging Face to explore how the
|
| 165 |
-
different terms contribute to the prediction.
|
| 166 |
</p>
|
| 167 |
|
| 168 |
<p>
|
| 169 |
-
|
| 170 |
-
<b>NVIDIA H100 GPU</b> with <b>NVIDIA TensorRT-LLM</b> under the
|
| 171 |
-
following experimental configuration:
|
| 172 |
-
</p>
|
| 173 |
-
|
| 174 |
<ul>
|
| 175 |
<li><code>max_batch_size = 1024</code></li>
|
| 176 |
<li><code>max_num_tokens = 8192</code></li>
|
| 177 |
<li><code>num_requests = 1000</code></li>
|
| 178 |
</ul>
|
| 179 |
|
| 180 |
-
|
| 181 |
-
Future releases will extend the analysis to additional hardware
|
| 182 |
-
platforms and inference frameworks.
|
| 183 |
</p>
|
| 184 |
|
| 185 |
<p>
|
| 186 |
-
|
| 187 |
-
|
| 188 |
-
|
| 189 |
-
|
| 190 |
-
Energy Efficiency of LLM Inference
|
| 191 |
-
</a>.
|
| 192 |
</p>
|
| 193 |
|
| 194 |
-
<h4>Validated models</h4>
|
| 195 |
|
| 196 |
-
<
|
| 197 |
-
|
|
|
|
| 198 |
|
| 199 |
-
<
|
|
|
|
| 200 |
|
| 201 |
-
<
|
| 202 |
-
|
| 203 |
-
|
| 204 |
-
placeholder="tiiuae/falcon-rw-7b"
|
| 205 |
-
>
|
| 206 |
-
<button type="button" onclick="loadModel()">Load model</button>
|
| 207 |
|
| 208 |
-
<
|
|
|
|
|
|
|
| 209 |
|
| 210 |
-
<
|
|
|
|
|
|
|
|
|
|
| 211 |
|
| 212 |
-
<
|
|
|
|
|
|
|
| 213 |
|
| 214 |
-
|
| 215 |
-
<label for="d">Hidden size (<i>d</i>)</label>
|
| 216 |
-
<input id="d" type="number" value="4096">
|
| 217 |
-
</div>
|
| 218 |
|
| 219 |
-
|
| 220 |
-
<label for="L">Layers (<i>L</i>)</label>
|
| 221 |
-
<input id="L" type="number" value="36">
|
| 222 |
-
</div>
|
| 223 |
|
| 224 |
-
|
| 225 |
-
<label for="nq">Query heads (<i>n</i><sub>q</sub>)</label>
|
| 226 |
-
<input id="nq" type="number" value="64">
|
| 227 |
-
</div>
|
| 228 |
|
| 229 |
-
|
| 230 |
-
|
| 231 |
-
|
| 232 |
-
|
| 233 |
|
|
|
|
|
|
|
|
|
|
| 234 |
</div>
|
| 235 |
|
| 236 |
-
<div
|
|
|
|
|
|
|
|
|
|
| 237 |
|
| 238 |
-
|
| 239 |
-
|
| 240 |
-
|
| 241 |
-
|
| 242 |
|
| 243 |
-
|
| 244 |
-
</div>
|
| 245 |
|
| 246 |
-
<div
|
| 247 |
-
|
| 248 |
-
|
| 249 |
-
|
| 250 |
-
|
| 251 |
-
|
| 252 |
-
|
| 253 |
-
|
| 254 |
-
|
| 255 |
-
|
| 256 |
-
|
| 257 |
-
|
| 258 |
-
|
| 259 |
-
|
| 260 |
-
|
| 261 |
-
|
| 262 |
-
|
| 263 |
-
|
| 264 |
-
|
| 265 |
-
|
| 266 |
-
|
| 267 |
-
|
| 268 |
-
|
| 269 |
-
|
| 270 |
-
|
| 271 |
-
|
| 272 |
-
</
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
| 273 |
</div>
|
| 274 |
|
| 275 |
-
<!-- app.js reads the actual integer token count from here -->
|
| 276 |
<input id="nin" type="hidden" value="64">
|
| 277 |
-
|
| 278 |
</div>
|
| 279 |
|
| 280 |
-
<
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
| 281 |
|
| 282 |
<script>
|
| 283 |
const MAX_TOKENS = 4096;
|
|
|
|
| 284 |
|
| 285 |
function updateNin(exponent) {
|
| 286 |
-
const value = Math.round(2
|
| 287 |
const clamped = Math.min(MAX_TOKENS, Math.max(1, value));
|
| 288 |
|
| 289 |
-
document.getElementById(
|
| 290 |
-
document.getElementById(
|
|
|
|
|
|
|
| 291 |
}
|
| 292 |
</script>
|
| 293 |
|
|
|
|
|
|
|
|
|
|
| 294 |
<script src="app.js"></script>
|
| 295 |
|
| 296 |
</body>
|
| 297 |
-
</html>
|
|
|
|
| 1 |
<!DOCTYPE html>
|
| 2 |
+
<html>
|
|
|
|
|
|
|
| 3 |
<head>
|
| 4 |
<meta charset="UTF-8">
|
|
|
|
| 5 |
<title>SweetSpot v2 — Energy-per-Token demo</title>
|
| 6 |
|
| 7 |
+
<script src="https://cdn.plot.ly/plotly-latest.min.js"></script>
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
| 8 |
|
| 9 |
+
<style>
|
| 10 |
+
body {
|
| 11 |
+
font-family: sans-serif;
|
| 12 |
+
max-width: 1000px;
|
| 13 |
+
margin: auto;
|
| 14 |
+
padding: 20px;
|
| 15 |
+
}
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
| 16 |
|
| 17 |
+
.row {
|
| 18 |
+
display: flex;
|
| 19 |
+
gap: 20px;
|
| 20 |
+
flex-wrap: wrap;
|
| 21 |
}
|
|
|
|
| 22 |
|
| 23 |
+
label {
|
| 24 |
+
display: block;
|
| 25 |
+
margin-top: 10px;
|
| 26 |
}
|
| 27 |
|
| 28 |
+
input {
|
| 29 |
+
width: 120px;
|
| 30 |
}
|
|
|
|
|
|
|
|
|
|
| 31 |
|
| 32 |
+
button {
|
| 33 |
+
margin-top: 10px;
|
| 34 |
+
padding: 8px 15px;
|
| 35 |
+
}
|
| 36 |
+
</style>
|
| 37 |
</head>
|
| 38 |
|
| 39 |
<body>
|
| 40 |
|
| 41 |
+
<h2>
|
| 42 |
+
SweetSpot v2 — Energy-per-Token demo
|
| 43 |
+
</h2>
|
| 44 |
|
| 45 |
<p>
|
| 46 |
+
This interactive demo visualizes our analytical model for predicting the <b>energy consumption per generated token</b> as a function of the number of output tokens (<i>n</i><sub>out</sub>).
|
| 47 |
+
Select one of the validated models used in the paper or load the architecture of any <b>dense decoder-only Transformer</b> (up to approximately 20B parameters) from Hugging Face to explore how the different terms contribute to the prediction.
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
| 48 |
</p>
|
| 49 |
|
| 50 |
<p>
|
| 51 |
+
The model was fitted using empirical measurements collected on an <b>NVIDIA H100 GPU</b> with <b>NVIDIA TensorRT-LLM</b> under the following experimental configuration:
|
|
|
|
|
|
|
|
|
|
|
|
|
| 52 |
<ul>
|
| 53 |
<li><code>max_batch_size = 1024</code></li>
|
| 54 |
<li><code>max_num_tokens = 8192</code></li>
|
| 55 |
<li><code>num_requests = 1000</code></li>
|
| 56 |
</ul>
|
| 57 |
|
| 58 |
+
Future releases will extend the analysis to additional hardware platforms and inference frameworks.
|
|
|
|
|
|
|
| 59 |
</p>
|
| 60 |
|
| 61 |
<p>
|
| 62 |
+
For the complete methodology, experimental validation, and citation information, please see the paper:
|
| 63 |
+
<a href="YOUR_PAPER_URL" target="_blank">
|
| 64 |
+
SweetSpot v2: An Architecture-Aware Analytical Model for Predicting Energy Efficiency of LLM Inference
|
| 65 |
+
</a>.
|
|
|
|
|
|
|
| 66 |
</p>
|
| 67 |
|
|
|
|
| 68 |
|
| 69 |
+
<h4>
|
| 70 |
+
Validated models:
|
| 71 |
+
</h4>
|
| 72 |
|
| 73 |
+
<select id="preset">
|
| 74 |
+
</select>
|
| 75 |
|
| 76 |
+
<button onclick="loadPreset()">
|
| 77 |
+
Load selected model
|
| 78 |
+
</button>
|
|
|
|
|
|
|
|
|
|
| 79 |
|
| 80 |
+
<h4>
|
| 81 |
+
Or load another public Hugging Face model (dense, < 20B):
|
| 82 |
+
</h4>
|
| 83 |
|
| 84 |
+
<input
|
| 85 |
+
id="repo"
|
| 86 |
+
style="width:400px"
|
| 87 |
+
placeholder="tiiuae/falcon-rw-7b">
|
| 88 |
|
| 89 |
+
<button onclick="loadModel()">
|
| 90 |
+
Load model
|
| 91 |
+
</button>
|
| 92 |
|
| 93 |
+
<p id="status"></p>
|
|
|
|
|
|
|
|
|
|
| 94 |
|
| 95 |
+
<h4>Architecture parameters</h4>
|
|
|
|
|
|
|
|
|
|
| 96 |
|
| 97 |
+
<div class="row">
|
|
|
|
|
|
|
|
|
|
| 98 |
|
| 99 |
+
<div>
|
| 100 |
+
<label>Hidden size (d):</label>
|
| 101 |
+
<input id="d" type="number" value="4096">
|
| 102 |
+
</div>
|
| 103 |
|
| 104 |
+
<div>
|
| 105 |
+
<label>Layers (L):</label>
|
| 106 |
+
<input id="L" type="number" value="36">
|
| 107 |
</div>
|
| 108 |
|
| 109 |
+
<div>
|
| 110 |
+
<label>Query heads (<i>n</i><sub>q</sub>):</label>
|
| 111 |
+
<input id="nq" type="number" value="64">
|
| 112 |
+
</div>
|
| 113 |
|
| 114 |
+
<div>
|
| 115 |
+
<label>KV heads (<i>n</i><sub>kv</sub>):</label>
|
| 116 |
+
<input id="nkv" type="number" value="64">
|
| 117 |
+
</div>
|
| 118 |
|
| 119 |
+
</div>
|
|
|
|
| 120 |
|
| 121 |
+
<div id="ninWrapper">
|
| 122 |
+
<label>
|
| 123 |
+
Input Tokens (<i>n</i><sub>in</sub>):
|
| 124 |
+
<span id="ninValue">64</span>
|
| 125 |
+
</label>
|
| 126 |
+
|
| 127 |
+
<input
|
| 128 |
+
id="ninSlider"
|
| 129 |
+
type="range"
|
| 130 |
+
min="0"
|
| 131 |
+
max="12"
|
| 132 |
+
step="0.01"
|
| 133 |
+
value="6"
|
| 134 |
+
list="ninTicks"
|
| 135 |
+
oninput="updateNin(this.value)">
|
| 136 |
+
|
| 137 |
+
<datalist id="ninTicks">
|
| 138 |
+
<option value="0"></option>
|
| 139 |
+
<option value="1"></option>
|
| 140 |
+
<option value="2"></option>
|
| 141 |
+
<option value="3"></option>
|
| 142 |
+
<option value="4"></option>
|
| 143 |
+
<option value="5"></option>
|
| 144 |
+
<option value="6"></option>
|
| 145 |
+
<option value="7"></option>
|
| 146 |
+
<option value="8"></option>
|
| 147 |
+
<option value="9"></option>
|
| 148 |
+
<option value="10"></option>
|
| 149 |
+
<option value="11"></option>
|
| 150 |
+
<option value="12"></option>
|
| 151 |
+
</datalist>
|
| 152 |
+
|
| 153 |
+
<div id="ninLabels">
|
| 154 |
+
<span>1</span><span>2</span><span>4</span><span>8</span>
|
| 155 |
+
<span>16</span><span>32</span><span>64</span><span>128</span>
|
| 156 |
+
<span>256</span><span>512</span><span>1024</span><span>2048</span><span>4096</span>
|
| 157 |
</div>
|
| 158 |
|
|
|
|
| 159 |
<input id="nin" type="hidden" value="64">
|
|
|
|
| 160 |
</div>
|
| 161 |
|
| 162 |
+
<style>
|
| 163 |
+
#ninSlider {
|
| 164 |
+
width: 100%;
|
| 165 |
+
box-sizing: border-box;
|
| 166 |
+
}
|
| 167 |
+
|
| 168 |
+
#ninLabels {
|
| 169 |
+
display: flex;
|
| 170 |
+
justify-content: space-between;
|
| 171 |
+
padding: 0 8px;
|
| 172 |
+
font-size: 12px;
|
| 173 |
+
color: #555;
|
| 174 |
+
}
|
| 175 |
+
</style>
|
| 176 |
|
| 177 |
<script>
|
| 178 |
const MAX_TOKENS = 4096;
|
| 179 |
+
const MAX_EXPONENT = 12; // log2(4096)
|
| 180 |
|
| 181 |
function updateNin(exponent) {
|
| 182 |
+
const value = Math.round(Math.pow(2, parseFloat(exponent)));
|
| 183 |
const clamped = Math.min(MAX_TOKENS, Math.max(1, value));
|
| 184 |
|
| 185 |
+
document.getElementById('nin').value = clamped;
|
| 186 |
+
document.getElementById('ninValue').textContent = clamped;
|
| 187 |
+
|
| 188 |
+
draw();
|
| 189 |
}
|
| 190 |
</script>
|
| 191 |
|
| 192 |
+
<div id="plot"></div>
|
| 193 |
+
|
| 194 |
+
|
| 195 |
<script src="app.js"></script>
|
| 196 |
|
| 197 |
</body>
|
| 198 |
+
</html>
|