Humuhumu33 commited on
Commit
e4b6a8f
·
verified ·
1 Parent(s): 7efaf4b

warmup uses decode() to precompile batched pipelines (cut first-msg TTFT)

Browse files
Files changed (1) hide show
  1. index.html +3 -1
index.html CHANGED
@@ -115,7 +115,9 @@ async function warmUp() {
115
  const gpu = engine && engine._gpu;
116
  if (!gpu || !gpu.sync || !gpu.generate || !gpu.reset) return;
117
  try {
118
- gpu.reset(); await gpu.generate(engine.tokenize("Hello there."), 24, m.rep ?? 1.3); gpu.reset(); // boost clock (throwaway)
 
 
119
  let primeIds = engine.tokenize(frameSystem());
120
  if (m.bos && engine.bosId != null) primeIds = [engine.bosId, ...primeIds];
121
  await gpu.sync(primeIds, true); // prefill the system prompt into KV; leaves cache == primeIds
 
115
  const gpu = engine && engine._gpu;
116
  if (!gpu || !gpu.sync || !gpu.generate || !gpu.reset) return;
117
  try {
118
+ // throwaway via decode() (not generate()) so it BUILDS the batched-decode pipelines now — else the first real
119
+ // message pays ~1.7s of one-time shader compilation. Also boosts the GPU clock.
120
+ gpu.reset(); await (gpu.decode || gpu.generate)(engine.tokenize("Hello there."), 24, m.rep ?? 1.3); gpu.reset();
121
  let primeIds = engine.tokenize(frameSystem());
122
  if (m.bos && engine.bosId != null) primeIds = [engine.bosId, ...primeIds];
123
  await gpu.sync(primeIds, true); // prefill the system prompt into KV; leaves cache == primeIds