Spaces:
Running
Running
spec-decode for BitNet: subNorm+f32-KV batched verify, ?spec + ?bench=spec
Browse files- core/engine.js +1 -1
core/engine.js
CHANGED
|
@@ -80,7 +80,7 @@ export async function createEngine(modelEntry, loaded) {
|
|
| 80 |
// head, the drafter proposes and the target batch-verifies — output is BYTE-IDENTICAL to greedy decode
|
| 81 |
// (greedy verify), streamed via onCommit. Any incompatibility/throw falls straight through to the standard
|
| 82 |
// loop below, so default Q (no drafter) and unsupported models are completely unaffected.
|
| 83 |
-
if (!err && _drafter && gpu.specDecode && gpu.setDrafter) {
|
| 84 |
try {
|
| 85 |
gpu.setDrafter(_drafter);
|
| 86 |
const out = []; const t0 = _perf(); let ttft2 = 0, tokps2 = 0;
|
|
|
|
| 80 |
// head, the drafter proposes and the target batch-verifies — output is BYTE-IDENTICAL to greedy decode
|
| 81 |
// (greedy verify), streamed via onCommit. Any incompatibility/throw falls straight through to the standard
|
| 82 |
// loop below, so default Q (no drafter) and unsupported models are completely unaffected.
|
| 83 |
+
if (!err && (_drafter || globalThis.__spec) && gpu.specDecode && gpu.setDrafter) {
|
| 84 |
try {
|
| 85 |
gpu.setDrafter(_drafter);
|
| 86 |
const out = []; const t0 = _perf(); let ttft2 = 0, tokps2 = 0;
|