Spaces:
Running
Running
warmup uses decode() to precompile batched pipelines (cut first-msg TTFT)
Browse files- index.html +3 -1
index.html
CHANGED
|
@@ -115,7 +115,9 @@ async function warmUp() {
|
|
| 115 |
const gpu = engine && engine._gpu;
|
| 116 |
if (!gpu || !gpu.sync || !gpu.generate || !gpu.reset) return;
|
| 117 |
try {
|
| 118 |
-
|
|
|
|
|
|
|
| 119 |
let primeIds = engine.tokenize(frameSystem());
|
| 120 |
if (m.bos && engine.bosId != null) primeIds = [engine.bosId, ...primeIds];
|
| 121 |
await gpu.sync(primeIds, true); // prefill the system prompt into KV; leaves cache == primeIds
|
|
|
|
| 115 |
const gpu = engine && engine._gpu;
|
| 116 |
if (!gpu || !gpu.sync || !gpu.generate || !gpu.reset) return;
|
| 117 |
try {
|
| 118 |
+
// throwaway via decode() (not generate()) so it BUILDS the batched-decode pipelines now — else the first real
|
| 119 |
+
// message pays ~1.7s of one-time shader compilation. Also boosts the GPU clock.
|
| 120 |
+
gpu.reset(); await (gpu.decode || gpu.generate)(engine.tokenize("Hello there."), 24, m.rep ?? 1.3); gpu.reset();
|
| 121 |
let primeIds = engine.tokenize(frameSystem());
|
| 122 |
if (m.bos && engine.bosId != null) primeIds = [engine.bosId, ...primeIds];
|
| 123 |
await gpu.sync(primeIds, true); // prefill the system prompt into KV; leaves cache == primeIds
|