Spaces:
Running
Running
NakliTechie commited on
Commit Β·
f4899fa
1
Parent(s): 2aae41c
Sync from GitHub 2026-10-22T06:17:49Z
Browse files- index.html +950 -30
index.html
CHANGED
|
@@ -2682,6 +2682,7 @@
|
|
| 2682 |
<li><strong>Gemma 4 26B-A4B · SSD-streamed (experimental)</strong> (~14 GB) — text, with reasoning. Google's Gemma 4 26B-A4B mixture-of-experts (128 experts, 8 active per token) from its official 4-bit QAT GGUF (14.4 GB) on the same from-scratch WebGPU engine family: attention, the dense MLP, routers and KV cache stay on the GPU (~1.6 GB plus a 4 GB expert pool) while the 3,840 routed experts stream from on-device storage (OPFS) as the router picks them. The first load copies the model into this browser's storage once (~14 GB of disk; about 12 minutes on a fast connection). About 23 tokens/s on an M4 Pro with 24 GB, and prompts are read in chunks at about 55 tokens/s. Greedy output matches llama.cpp. Flip <strong>Show reasoning</strong> for its thinking.</li>
|
| 2683 |
<li><strong>LFM2.5 24B-A2B · SSD-streamed (experimental)</strong> (~13.6 GB) — text, reasoning-only. Liquid AI's LFM2.5-24B-A2B (released 2026-10-13): a 24B mixture-of-experts (64 experts, 4 active per token, about 2.4B parameters per token) with short-convolution and attention layers, from Liquid's own 4-bit GGUF. Short-convolution and attention layers, routers and KV cache stay on the GPU while the 2,432 routed experts stream from on-device storage (OPFS) as the router picks them. The first load copies the model into this browser's storage once (~13.6 GB of disk). About 16 tokens/s on an M4 Pro with 24 GB, and prompts are read in chunks at about 50 tokens/s. Greedy output matches llama.cpp (8 of 8 prompts). It always reasons before it answers, so every reply opens with its reasoning. It can use tools (calculator, time, memory, reminders): it called the right tool on 6 of 6 test prompts and answered 5 of 5 plain questions without one.</li>
|
| 2684 |
<li><strong>Flint 40B-A2.5B · SSD-streamed (experimental)</strong> (~26 GB) — text, coding, reasoning. Recursive's Flint (released 2026-10-21, Apache-2.0): a 39.8B mixture-of-experts coding model (256 experts, 6 active per token, about 2.5B parameters per token), from Recursive's own 4-bit (Q4_1) GGUF, whose card asks for 48 GB of memory under llama.cpp. Attention, routers, the token table and KV cache stay on the GPU while the 12,032 routed experts stream from on-device storage (OPFS) as the router picks them. The first load copies the model into this browser's storage once (~26 GB of disk). About 8.7 tokens/s on an M4 Pro with 24 GB, and prompts are read at about 27 tokens/s. Its output stays as close to llama.cpp's Metal backend as llama.cpp's own CPU backend does. It reasons before it answers, so replies open with its reasoning. Tools are off for now.</li>
|
|
|
|
| 2685 |
</ul>
|
| 2686 |
</details>
|
| 2687 |
</div>
|
|
@@ -2998,6 +2999,7 @@
|
|
| 2998 |
<option value="gemma4-26b-a4b-ssd">Gemma 4 26B-A4B · SSD-streamed, experimental (~14 GB)</option>
|
| 2999 |
<option value="lfm25-24b-a2b-ssd">LFM2.5 24B-A2B · SSD-streamed, experimental (~13.6 GB)</option>
|
| 3000 |
<option value="flint-40b-a2b-ssd">Flint 40B-A2.5B · SSD-streamed, experimental, coding (~26 GB)</option>
|
|
|
|
| 3001 |
</select>
|
| 3002 |
<!-- Model picker (Chunk L layer 2): renders #modelSelect's options grouped by what is
|
| 3003 |
already in this browser; picking sets the select's value and fires its change event. -->
|
|
@@ -8595,6 +8597,8 @@ self.onmessage = async (e) => {
|
|
| 8595 |
if (loaded != null && total) post({ type: 'progress', data: { status: 'loading', file: `weights to GPU (${loaded} of ${total} tensors)` } });
|
| 8596 |
} else if (ev.status === 'weights') {
|
| 8597 |
if (typeof ev.loaded === 'number' && ev.total > 0) post({ type: 'progress', data: { status: 'progress_total', file: WEIGHTS_LABEL, loaded: ev.loaded, total: ev.total } });
|
|
|
|
|
|
|
| 8598 |
} else if (ev.status === 'init') {
|
| 8599 |
post({ type: 'progress', data: { status: 'initiate', file: 'WebGPU device' } });
|
| 8600 |
}
|
|
@@ -9383,6 +9387,24 @@ self.addEventListener('message', async (e) => {
|
|
| 9383 |
contextSize: 8192,
|
| 9384 |
genConfig: { max_new_tokens: 4096 },
|
| 9385 |
},
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
| 9386 |
'minicpm5-2b-gguf': {
|
| 9387 |
id: 'https://huggingface.co/openbmb/MiniCPM5-2B-GGUF/resolve/main/MiniCPM5-2B-Q4_K_M.gguf',
|
| 9388 |
label: 'MiniCPM5 2B Β· GGUF',
|
|
@@ -17700,6 +17722,7 @@ self.addEventListener('message', async (e) => {
|
|
| 17700 |
'gemma4moe-ssd': { engine: 'gemma4_moe_ssd.js', exportName: 'Gemma4MoeSsd', label: 'Gemma 4 26B-A4B (Q4_0)' },
|
| 17701 |
'lfm2moe-ssd': { engine: 'lfm2_moe_ssd.js', exportName: 'Lfm2MoeSsd', label: 'LFM2.5 24B-A2B (Q4_0)' },
|
| 17702 |
'flint-ssd': { engine: 'flint_moe_ssd.js', exportName: 'FlintMoeSsd', label: 'Flint 40B-A2.5B (Q4_1)' },
|
|
|
|
| 17703 |
'hyv3-ssd': { engine: 'hy_v3_moe_ssd.js', exportName: 'HyV3MoeSsd', label: 'Hy-Memory 30B-A3B (Q8_0)' }, // the memory writer only (WRITER_MODELS)
|
| 17704 |
'qwen35dense': { engine: 'qwen35_dense.js', exportName: 'Qwen35Dense', label: 'Underdog Saluki 27B (IQ2-mix)' },
|
| 17705 |
})[backend] || null;
|
|
@@ -19314,7 +19337,7 @@ self.addEventListener('message', async (e) => {
|
|
| 19314 |
const ENGINE_STORES = {
|
| 19315 |
'transformers': ['transformers-cache'], 'wllama': ['wllama'], 'lfm2-webgpu': ['gguf-v1'],
|
| 19316 |
'bonsai2-webgpu': ['gguf-cache-v1'], 'gemma4-webgpu': ['safetensors-cache-v1', 'bonsai-pipeline-v1'],
|
| 19317 |
-
'qwen35moe-ssd': ['on disk (OPFS)'], 'minicpmv-ssd': ['on disk (OPFS)'], 'gemma4moe-ssd': ['on disk (OPFS)'], 'lfm2moe-ssd': ['on disk (OPFS)'], 'flint-ssd': ['on disk (OPFS)'], 'qwen35dense': ['on disk (OPFS)'],
|
| 19318 |
};
|
| 19319 |
function parseSizeBytes(size) {
|
| 19320 |
const m = /([\d.]+)\s*(GB|MB)/i.exec(String(size || ''));
|
|
@@ -23931,6 +23954,704 @@ export function codedError(e, { signal, fallback = 'LOAD_FAILED', what = 'load'
|
|
| 23931 |
return new DiskformerError(fallback, `${what} failed: ${(e && e.message) || e}`, { cause: e });
|
| 23932 |
}
|
| 23933 |
</script>
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
| 23934 |
<script type="text/plain" data-module="flint_moe_ssd.js">/* flint_moe_ssd.js β Recursive Flint (GGUF arch `flint`, 39.8B total, ~2.5B active) with its routed experts on disk:
|
| 23935 |
* rung 2g of the SSD-streaming ladder. Flint is a coding-agent model (released 2026-10-21, Apache-2.0). As an engine it
|
| 23936 |
* is the Hy-Memory graph (hy_v3_moe_ssd.js, on the LFM2.5 rung) with attention sinks, a sliding window and no RoPE on
|
|
@@ -32240,6 +32961,54 @@ var<workgroup> red: array<f32, ${MV_ROWS * 64}>;
|
|
| 32240 |
}`;
|
| 32241 |
}
|
| 32242 |
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
| 32243 |
// Token-major batch: y[tΒ·M + m] = Ξ£_k x[tΒ·N + k]Β·W[m][k]. A 256-thread workgroup computes a 64-row Γ 64-token tile.
|
| 32244 |
// Per 32-wide k chunk every thread dequantizes 8 weights (64 rows Γ 4 groups) and loads 8 activations into shared
|
| 32245 |
// memory (k-major f32 tiles, 16 KB in all), then accumulates a 4-row Γ 4-token block: each weight read from
|
|
@@ -40204,6 +40973,9 @@ export function createGemmaPle({ key = null, slots = 32768, warm = GEMMA4_WARM,
|
|
| 40204 |
<script type="text/plain" data-module="qwen35_dense.js">/* qwen35_dense.js β dense Qwen3.5-family models (GGUF arch `qwen35`, e.g. Qwen3.8-27B) from an imatrix
|
| 40205 |
* IQ-mix GGUF, in a browser tab. First user: Underdog Saluki 27B 1.0 (ConwayResearch, Apache-2.0), one 7.9 GB
|
| 40206 |
* GGUF mixing eleven llama.cpp block types.
|
|
|
|
|
|
|
|
|
|
| 40207 |
*
|
| 40208 |
* The trunk is the one qwen35_moe_ssd.js runs (llama.cpp's qwen35 graph): 3 of every 4 layers are Gated DeltaNet,
|
| 40209 |
* every 4th is gated full attention with q/k RMSNorm and partial NeoX RoPE; this module reuses those kernels and
|
|
@@ -40220,7 +40992,8 @@ export function createGemmaPle({ key = null, slots = 32768, warm = GEMMA4_WARM,
|
|
| 40220 |
|
| 40221 |
import { Qwen3MoeSsd, STORAGE, COPY_SRC, COPY_DST, MAP_READ, UNIFORM } from './qwen3_moe_ssd.js';
|
| 40222 |
import { QWEN35_KERNELS, QWEN35_BATCH_KERNELS } from './qwen35_moe_ssd.js';
|
| 40223 |
-
import {
|
|
|
|
| 40224 |
|
| 40225 |
export const UNDERDOG_SALUKI_27B = {
|
| 40226 |
repo: 'ConwayResearch/Underdog-Saluki-27B-1.0',
|
|
@@ -40230,18 +41003,21 @@ export const UNDERDOG_SALUKI_27B = {
|
|
| 40230 |
size: 7898369152,
|
| 40231 |
};
|
| 40232 |
const F32 = 0;
|
|
|
|
| 40233 |
|
| 40234 |
export function configFromGguf(kv) {
|
| 40235 |
const a = kv['general.architecture'];
|
| 40236 |
-
if (a !== 'qwen35') throw new Error(`Qwen35Dense needs a qwen35 GGUF, got ${a}`);
|
| 40237 |
-
const g = (k) => kv[`${a}.${k}`];
|
| 40238 |
return {
|
| 40239 |
arch: a,
|
| 40240 |
layers: g('block_count') - (g('nextn_predict_layers') || 0), hidden: g('embedding_length'), ff: g('feed_forward_length'),
|
| 40241 |
heads: g('attention.head_count'), kvHeads: g('attention.head_count_kv'), headDim: g('attention.key_length'),
|
| 40242 |
-
ropeDims: g('rope.dimension_count'), ropeTheta: g('rope.freq_base'), eps: g('attention.layer_norm_rms_epsilon'),
|
| 40243 |
-
attnInterval: g('full_attention_interval') || 4, contextLength: g('context_length'),
|
| 40244 |
-
|
|
|
|
|
|
|
| 40245 |
vocab: kv['tokenizer.ggml.tokens'].length,
|
| 40246 |
bos: kv['tokenizer.ggml.bos_token_id'], eos: kv['tokenizer.ggml.eos_token_id'],
|
| 40247 |
};
|
|
@@ -40291,8 +41067,54 @@ struct P { F: u32 }
|
|
| 40291 |
@compute @workgroup_size(256) fn main(@builtin(global_invocation_id) gi: vec3<u32>) {
|
| 40292 |
let i = gi.x; if (i >= p.F) { return; }
|
| 40293 |
let g = gu[i]; act[i] = (g / (1.0 + exp(-g))) * gu[p.F + i];
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
| 40294 |
}`,
|
| 40295 |
};
|
|
|
|
|
|
|
| 40296 |
|
| 40297 |
// Prefill kernels that replace qwen35_moe_ssd.js's batched DeltaNet step and attention (same bindings, same math):
|
| 40298 |
// gdnStepR β the gated delta rule over T tokens with each head's 128Γ128 state in registers (256 threads, 64
|
|
@@ -40427,11 +41249,12 @@ export class Qwen35Dense extends Qwen3MoeSsd {
|
|
| 40427 |
constructor(device, manifest, gguf, opts) {
|
| 40428 |
super(device, manifest, gguf, opts);
|
| 40429 |
const c = this.cfg;
|
| 40430 |
-
if (c.ssm.dConv !== 4) throw new Error(`gdnConv assumes a 4-tap conv, got ${c.ssm.dConv}`);
|
| 40431 |
-
if (c.ssm.dState !== 128 || c.ssm.dInner !== c.ssm.vHeads * c.ssm.dState) throw new Error('gdnStep assumes 128-wide heads');
|
| 40432 |
this.recurrent = Array.from({ length: c.layers }, (_, l) => (l + 1) % c.attnInterval !== 0);
|
| 40433 |
this.types = {};
|
| 40434 |
for (const [name, d] of Object.entries(manifest.dense.tensors)) this.types[name] = d.ggml === F32 ? null : QTYPE_BY_ID[d.ggml];
|
|
|
|
| 40435 |
this.prefetch = false;
|
| 40436 |
// Prompt chunks of up to 256 tokens. Without flash attention the batched scores take heads Γ chunk Γ maxCtx
|
| 40437 |
// floats; then the chunk shrinks to keep them near 96 MB.
|
|
@@ -40457,6 +41280,7 @@ export class Qwen35Dense extends Qwen3MoeSsd {
|
|
| 40457 |
const embedType = this.types['token_embd.weight'];
|
| 40458 |
for (const t of new Set(Object.values(this.types).filter(Boolean))) {
|
| 40459 |
k[`mv_${t}`] = mvKernel(t); k[`mm_${t}`] = this.matrixKernels ? mmSgKernel(t) : mmKernel(t);
|
|
|
|
| 40460 |
if (PADDED.has(t)) k[`repack_${t}`] = repackKernel(t);
|
| 40461 |
}
|
| 40462 |
k[`embed_${embedType}`] = embedKernel(embedType);
|
|
@@ -40507,7 +41331,7 @@ export class Qwen35Dense extends Qwen3MoeSsd {
|
|
| 40507 |
let pending = [];
|
| 40508 |
const flush = async () => { await dev.queue.onSubmittedWorkDone(); for (const b of pending) { this.gpuBytes.dense -= b.size; b.destroy(); } pending = []; };
|
| 40509 |
for (const [name, t] of Object.entries(this.types)) {
|
| 40510 |
-
if (!t || !PADDED.has(t)) continue;
|
| 40511 |
const dims = this.manifest.dense.tensors[name].dims, rows = dims.reduce((a, b) => a * b, 1) / dims[0];
|
| 40512 |
const words = gpuRowBytes(t, dims[0]) * rows / 4, src = this.w[name].raw;
|
| 40513 |
const dst = this.buffer(words * 4, STORAGE | COPY_DST, 'dense');
|
|
@@ -40530,7 +41354,7 @@ export class Qwen35Dense extends Qwen3MoeSsd {
|
|
| 40530 |
}
|
| 40531 |
|
| 40532 |
kvBytes() {
|
| 40533 |
-
const c = this.cfg, s = c.ssm, convDim = 2 * s.kHeads * s.dState + s.dInner;
|
| 40534 |
return this.recurrent.reduce((sum, r) => sum + (r ? 3 * convDim * 4 + s.vHeads * s.dState * s.dState * 4 : 2 * 2 * this.maxCtx * c.kvHeads * c.headDim), 0);
|
| 40535 |
}
|
| 40536 |
|
|
@@ -40544,11 +41368,18 @@ export class Qwen35Dense extends Qwen3MoeSsd {
|
|
| 40544 |
mm(name, x, y, M, N) {
|
| 40545 |
const t = this.types[name], w = this.w[name];
|
| 40546 |
const u = this.uniform([M, N, gpuRowBytes(t, N), 0]);
|
| 40547 |
-
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
| 40548 |
}
|
| 40549 |
|
| 40550 |
initBuffers() {
|
| 40551 |
-
const c = this.cfg, s = c.ssm;
|
|
|
|
| 40552 |
const H = c.hidden, hd = c.headDim, QN = c.heads * hd, kvn = c.kvHeads * hd, F = c.ff;
|
| 40553 |
const keyDim = s.kHeads * s.dState, convDim = 2 * keyDim + s.dInner;
|
| 40554 |
if ((F * 4) % 256) throw new Error('gate/up halves must be 256-byte aligned');
|
|
@@ -40576,7 +41407,7 @@ export class Qwen35Dense extends Qwen3MoeSsd {
|
|
| 40576 |
const U = (w) => this.uniform(w), eps = { f: c.eps };
|
| 40577 |
const u = {
|
| 40578 |
nH: U([H]), rmsH: U([H, eps]),
|
| 40579 |
-
ropeQ: U([hd, 2 * hd, c.ropeDims, 1, eps]), ropeK: U([hd, hd, c.ropeDims, 0, eps]), kvn: U([kvn]),
|
| 40580 |
att: U([c.heads, c.kvHeads, hd, { f: 1 / Math.sqrt(hd) }]), attOut: U([c.heads, c.kvHeads, hd, 0]), gate: U([QN]),
|
| 40581 |
conv: U([convDim]), step: U([s.kHeads, s.dState, keyDim, 0, eps, { f: 1 / Math.sqrt(s.dState) }]), ng: U([s.dState, eps]),
|
| 40582 |
silu: U([F]), am: U([c.vocab]),
|
|
@@ -40607,7 +41438,7 @@ export class Qwen35Dense extends Qwen3MoeSsd {
|
|
| 40607 |
});
|
| 40608 |
} else {
|
| 40609 |
Object.assign(g, {
|
| 40610 |
-
q: this.mv(n('attn_q.weight'), a.xn, a.qg, 2 * QN, H),
|
| 40611 |
k: this.mv(n('attn_k.weight'), a.xn, a.k, kvn, H),
|
| 40612 |
v: this.mv(n('attn_v.weight'), a.xn, a.v, kvn, H),
|
| 40613 |
ropeQ: this.bind('qkNormRopeP', [a.qg, raw(n('attn_q_norm.weight')), a.rope, a.q, a.gate, u.ropeQ]),
|
|
@@ -40616,13 +41447,13 @@ export class Qwen35Dense extends Qwen3MoeSsd {
|
|
| 40616 |
score: this.bind('attnScore', [a.q, this.kc[l], a.sc, this.tok, u.att]),
|
| 40617 |
soft: this.bind('softmax', [a.sc, this.tok]),
|
| 40618 |
attOut: this.bind('attnOut', [a.sc, this.vc[l], a.att, this.tok, u.attOut]),
|
| 40619 |
-
gate: this.bind('sigmoidMul', [a.att, a.gate, u.gate]),
|
| 40620 |
o: this.mv(n('attn_output.weight'), a.att, a.o, H, QN),
|
| 40621 |
});
|
| 40622 |
}
|
| 40623 |
Object.assign(g, {
|
| 40624 |
addO: this.bind('addInPlace', [a.x, a.o, u.nH]),
|
| 40625 |
-
rmsF: this.bind('rmsnorm', [a.x, raw(n(
|
| 40626 |
up: this.mv(n('ffn_up.weight'), a.xn, half(1), F, H),
|
| 40627 |
gt: this.mv(n('ffn_gate.weight'), a.xn, half(0), F, H),
|
| 40628 |
silu: this.bind('siluMul', [a.gu, a.act, u.silu]),
|
|
@@ -40637,7 +41468,7 @@ export class Qwen35Dense extends Qwen3MoeSsd {
|
|
| 40637 |
// ββ one token βββββββββββββββββββββββββββββββββββββββββββββββββββββββββββββ
|
| 40638 |
run(pass, op) { this.dispatch(pass, op.k, op.g, op.wg); }
|
| 40639 |
encodeLayer(pass, l, seqLen) {
|
| 40640 |
-
const c = this.cfg, s = c.ssm, g = this.layers[l], d = (nm, gr, x) => this.dispatch(pass, nm, gr, x);
|
| 40641 |
const H = c.hidden, QN = c.heads * c.headDim, kvn = c.kvHeads * c.headDim, convDim = 2 * s.kHeads * s.dState + s.dInner;
|
| 40642 |
d('rmsnorm', g.rmsA, 1);
|
| 40643 |
if (g.recur) {
|
|
@@ -40653,7 +41484,7 @@ export class Qwen35Dense extends Qwen3MoeSsd {
|
|
| 40653 |
d('attnScore', g.score, Math.ceil(c.heads * seqLen / 256));
|
| 40654 |
d('softmax', g.soft, c.heads);
|
| 40655 |
d('attnOut', g.attOut, Math.ceil(QN / 256));
|
| 40656 |
-
d('sigmoidMul', g.gate, Math.ceil(QN / 256));
|
| 40657 |
this.run(pass, g.o);
|
| 40658 |
}
|
| 40659 |
d('addInPlace', g.addO, Math.ceil(H / 256));
|
|
@@ -40662,6 +41493,8 @@ export class Qwen35Dense extends Qwen3MoeSsd {
|
|
| 40662 |
d('siluMul', g.silu, Math.ceil(c.ff / 256));
|
| 40663 |
this.run(pass, g.down);
|
| 40664 |
d('addInPlace', g.addD, Math.ceil(H / 256));
|
|
|
|
|
|
|
| 40665 |
}
|
| 40666 |
encodeHead(pass, want) {
|
| 40667 |
this.dispatch(pass, 'rmsnorm', this.g.rmsOut, 1);
|
|
@@ -40675,6 +41508,7 @@ export class Qwen35Dense extends Qwen3MoeSsd {
|
|
| 40675 |
if (this.position >= this.maxCtx) throw this.contextFull();
|
| 40676 |
const tStart = performance.now(), pos = this.position;
|
| 40677 |
this.writeTokenUniforms(token, pos);
|
|
|
|
| 40678 |
const enc = dev.createCommandEncoder(), pass = enc.beginComputePass();
|
| 40679 |
this.dispatch(pass, `embed_${this.types['token_embd.weight']}`, this.g.embed, Math.ceil(c.hidden / 8 / 256));
|
| 40680 |
for (let l = 0; l < c.layers; l++) this.encodeLayer(pass, l, pos + 1);
|
|
@@ -40711,7 +41545,7 @@ export class Qwen35Dense extends Qwen3MoeSsd {
|
|
| 40711 |
|
| 40712 |
// ββ batched prefill: chunks of T prompt tokens, token-major βββββββββββββββ
|
| 40713 |
initBatch() {
|
| 40714 |
-
const c = this.cfg, s = c.ssm, B = this.chunkTokens, a = this.a;
|
| 40715 |
const H = c.hidden, hd = c.headDim, QN = c.heads * hd, kvn = c.kvHeads * hd, F = c.ff;
|
| 40716 |
const keyDim = s.kHeads * s.dState, convDim = 2 * keyDim + s.dInner;
|
| 40717 |
const A = (n) => this.buffer(n * 4, STORAGE | COPY_SRC | COPY_DST, 'batch');
|
|
@@ -40724,13 +41558,30 @@ export class Qwen35Dense extends Qwen3MoeSsd {
|
|
| 40724 |
const U = (w) => this.uniform(w), eps = { f: c.eps }, qb = this.qb, W = this.w;
|
| 40725 |
const u = {
|
| 40726 |
nH: U([H]), rmsH: U([H, eps]),
|
| 40727 |
-
ropeQ: U([hd, 2 * hd, 2 * QN, QN, c.ropeDims, 1, eps]), ropeK: U([hd, hd, kvn, kvn, c.ropeDims, 0, eps]), kvn: U([kvn]),
|
| 40728 |
att: U([c.heads, c.kvHeads, hd, { f: 1 / Math.sqrt(hd) }]), attOut: U([c.heads, c.kvHeads, hd, 0]), gate: U([QN]),
|
| 40729 |
conv: U([convDim]), step: U([s.kHeads, s.dState, keyDim, convDim, s.vHeads, 0, eps, { f: 1 / Math.sqrt(s.dState) }]),
|
| 40730 |
ng: U([s.dState, s.dInner, eps]), ff: U([F]),
|
| 40731 |
};
|
| 40732 |
const raw = (name) => W[name].raw, et = this.types['token_embd.weight'];
|
| 40733 |
this.gB = { embed: this.bind(`embedB_${et}`, [W['token_embd.weight'].raw, this.grid, b.ids, b.x, U([H, gpuRowBytes(et, H)]), qb]) };
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
| 40734 |
for (let l = 0; l < c.layers; l++) {
|
| 40735 |
const n = (t) => `blk.${l}.${t}`, g = { rmsA: this.bind('rmsnormB', [b.x, raw(n('attn_norm.weight')), b.xn, u.rmsH]) };
|
| 40736 |
if (this.recurrent[l]) {
|
|
@@ -40744,7 +41595,7 @@ export class Qwen35Dense extends Qwen3MoeSsd {
|
|
| 40744 |
});
|
| 40745 |
} else {
|
| 40746 |
Object.assign(g, {
|
| 40747 |
-
q: this.mm(n('attn_q.weight'), b.xn, b.qg, 2 * QN, H), k: this.mm(n('attn_k.weight'), b.xn, b.k, kvn, H), v: this.mm(n('attn_v.weight'), b.xn, b.v, kvn, H),
|
| 40748 |
ropeQ: this.bind('qkNormRopePB', [b.qg, raw(n('attn_q_norm.weight')), b.rope, b.q, b.gate, u.ropeQ]),
|
| 40749 |
ropeK: this.bind('qkNormRopePB', [b.k, raw(n('attn_k_norm.weight')), b.rope, b.kr, b.dummy, u.ropeK]),
|
| 40750 |
kv: this.bind('kvStoreB', [b.kr, b.v, this.kc[l], this.vc[l], u.kvn, qb]),
|
|
@@ -40753,13 +41604,13 @@ export class Qwen35Dense extends Qwen3MoeSsd {
|
|
| 40753 |
soft: this.bind('softmaxB', [b.sc, qb]),
|
| 40754 |
attOut: this.bind('attnOutB', [b.sc, this.vc[l], b.att, u.attOut, qb]),
|
| 40755 |
}),
|
| 40756 |
-
gate: this.bind('sigmoidMulB', [b.att, b.gate, u.gate, qb]),
|
| 40757 |
o: this.mm(n('attn_output.weight'), b.att, b.o, H, QN),
|
| 40758 |
});
|
| 40759 |
}
|
| 40760 |
Object.assign(g, {
|
| 40761 |
addO: this.bind('addInPlaceB', [b.x, b.o, u.nH, qb]),
|
| 40762 |
-
rmsF: this.bind('rmsnormB', [b.x, raw(n(
|
| 40763 |
gt: this.mm(n('ffn_gate.weight'), b.xn, b.g, F, H), up: this.mm(n('ffn_up.weight'), b.xn, b.u, F, H),
|
| 40764 |
silu: this.bind('siluMul2B', [b.g, b.u, b.act, u.ff, qb]),
|
| 40765 |
down: this.mm(n('ffn_down.weight'), b.act, b.o, H, F),
|
|
@@ -40770,9 +41621,13 @@ export class Qwen35Dense extends Qwen3MoeSsd {
|
|
| 40770 |
}
|
| 40771 |
|
| 40772 |
encodeLayerB(pass, l, T, S) {
|
| 40773 |
-
const c = this.cfg, s = c.ssm, g = this.layers[l].B, d = (nm, gr, x, y) => this.dispatch(pass, nm, gr, x, y);
|
| 40774 |
const H = c.hidden, QN = c.heads * c.headDim, kvn = c.kvHeads * c.headDim, convDim = 2 * s.kHeads * s.dState + s.dInner;
|
| 40775 |
-
const tt = Math.ceil(T / (this.matrixKernels ? MM_SG_TOKENS : MM_TILE))
|
|
|
|
|
|
|
|
|
|
|
|
|
| 40776 |
d('rmsnormB', g.rmsA, T);
|
| 40777 |
if (this.recurrent[l]) {
|
| 40778 |
mm(g.qkv); mm(g.z); mm(g.beta); mm(g.alpha);
|
|
@@ -40786,7 +41641,7 @@ export class Qwen35Dense extends Qwen3MoeSsd {
|
|
| 40786 |
d('kvStoreB', g.kv, Math.ceil(kvn / 256), T);
|
| 40787 |
if (g.flash) d('flashAttnB', g.flash, c.kvHeads, Math.ceil(T / 4));
|
| 40788 |
else { d('attnScoreB', g.score, Math.ceil(c.heads * T * S / 256)); d('softmaxB', g.soft, c.heads * T); d('attnOutB', g.attOut, Math.ceil(QN / 256), T); }
|
| 40789 |
-
d('sigmoidMulB', g.gate, Math.ceil(QN / 256), T);
|
| 40790 |
mm(g.o);
|
| 40791 |
}
|
| 40792 |
d('addInPlaceB', g.addO, Math.ceil(H / 256), T);
|
|
@@ -40795,9 +41650,12 @@ export class Qwen35Dense extends Qwen3MoeSsd {
|
|
| 40795 |
d('siluMul2B', g.silu, Math.ceil(c.ff / 256), T);
|
| 40796 |
mm(g.down);
|
| 40797 |
d('addInPlaceB', g.addD, Math.ceil(H / 256), T);
|
|
|
|
|
|
|
| 40798 |
}
|
| 40799 |
|
| 40800 |
-
// T = ids.length tokens at this.position through every layer in one submit; want applies to the last token
|
|
|
|
| 40801 |
async prefillChunk(ids, want = 'none') {
|
| 40802 |
const c = this.cfg, dev = this.device, H = c.hidden, T = ids.length, pos0 = this.position, S = pos0 + T;
|
| 40803 |
if (T > this.chunkTokens) throw new Error(`chunk of ${T} > ${this.chunkTokens}`);
|
|
@@ -40815,9 +41673,18 @@ export class Qwen35Dense extends Qwen3MoeSsd {
|
|
| 40815 |
let pass = enc.beginComputePass();
|
| 40816 |
this.dispatch(pass, `embedB_${this.types['token_embd.weight']}`, this.gB.embed, Math.ceil(H / 8 / 256), T);
|
| 40817 |
for (let l = 0; l < c.layers; l++) this.encodeLayerB(pass, l, T, S);
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
| 40818 |
pass.end();
|
| 40819 |
enc.copyBufferToBuffer(this.b.x, (T - 1) * H * 4, this.a.x, 0, H * 4);
|
| 40820 |
-
if (want
|
|
|
|
| 40821 |
pass = enc.beginComputePass();
|
| 40822 |
this.encodeHead(pass, want);
|
| 40823 |
pass.end();
|
|
@@ -40827,12 +41694,26 @@ export class Qwen35Dense extends Qwen3MoeSsd {
|
|
| 40827 |
dev.queue.submit([enc.finish()]);
|
| 40828 |
this.position += T;
|
| 40829 |
this.cached.push(...ids);
|
| 40830 |
-
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
| 40831 |
this.counters.tokens += T;
|
| 40832 |
this.counters.wallMs += performance.now() - tStart;
|
| 40833 |
return result;
|
| 40834 |
}
|
| 40835 |
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
| 40836 |
// The recurrent state cannot rewind, so any change before the cached end restarts from token 0.
|
| 40837 |
async prefill(ids, want = 'argmax') {
|
| 40838 |
let common = 0;
|
|
@@ -40884,6 +41765,45 @@ export class Qwen35Dense extends Qwen3MoeSsd {
|
|
| 40884 |
async dispose() { await this.retire(); try { this.device.destroy(); } catch (_) {} }
|
| 40885 |
}
|
| 40886 |
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
| 40887 |
// For kernel tests (scripts/test-qwen35-dense-kernels.mjs).
|
| 40888 |
export { PREFILL_KERNELS as QWEN35_DENSE_PREFILL_KERNELS };
|
| 40889 |
</script>
|
|
|
|
| 2682 |
<li><strong>Gemma 4 26B-A4B · SSD-streamed (experimental)</strong> (~14 GB) — text, with reasoning. Google's Gemma 4 26B-A4B mixture-of-experts (128 experts, 8 active per token) from its official 4-bit QAT GGUF (14.4 GB) on the same from-scratch WebGPU engine family: attention, the dense MLP, routers and KV cache stay on the GPU (~1.6 GB plus a 4 GB expert pool) while the 3,840 routed experts stream from on-device storage (OPFS) as the router picks them. The first load copies the model into this browser's storage once (~14 GB of disk; about 12 minutes on a fast connection). About 23 tokens/s on an M4 Pro with 24 GB, and prompts are read in chunks at about 55 tokens/s. Greedy output matches llama.cpp. Flip <strong>Show reasoning</strong> for its thinking.</li>
|
| 2683 |
<li><strong>LFM2.5 24B-A2B · SSD-streamed (experimental)</strong> (~13.6 GB) — text, reasoning-only. Liquid AI's LFM2.5-24B-A2B (released 2026-10-13): a 24B mixture-of-experts (64 experts, 4 active per token, about 2.4B parameters per token) with short-convolution and attention layers, from Liquid's own 4-bit GGUF. Short-convolution and attention layers, routers and KV cache stay on the GPU while the 2,432 routed experts stream from on-device storage (OPFS) as the router picks them. The first load copies the model into this browser's storage once (~13.6 GB of disk). About 16 tokens/s on an M4 Pro with 24 GB, and prompts are read in chunks at about 50 tokens/s. Greedy output matches llama.cpp (8 of 8 prompts). It always reasons before it answers, so every reply opens with its reasoning. It can use tools (calculator, time, memory, reminders): it called the right tool on 6 of 6 test prompts and answered 5 of 5 plain questions without one.</li>
|
| 2684 |
<li><strong>Flint 40B-A2.5B · SSD-streamed (experimental)</strong> (~26 GB) — text, coding, reasoning. Recursive's Flint (released 2026-10-21, Apache-2.0): a 39.8B mixture-of-experts coding model (256 experts, 6 active per token, about 2.5B parameters per token), from Recursive's own 4-bit (Q4_1) GGUF, whose card asks for 48 GB of memory under llama.cpp. Attention, routers, the token table and KV cache stay on the GPU while the 12,032 routed experts stream from on-device storage (OPFS) as the router picks them. The first load copies the model into this browser's storage once (~26 GB of disk). About 8.7 tokens/s on an M4 Pro with 24 GB, and prompts are read at about 27 tokens/s. Its output stays as close to llama.cpp's Metal backend as llama.cpp's own CPU backend does. It reasons before it answers, so replies open with its reasoning. Tools are off for now.</li>
|
| 2685 |
+
<li><strong>Qwen3 4B · FlashMap speculative (experimental)</strong> (~5.2 GB) — text, reasoning. Qwen's Qwen3-4B (Q8_0 GGUF, 4.3 GB) with FlashMap, a speculative-decoding drafter by Shen et al. (arXiv:2610.24365; 0.95 GB) that proposes 16 tokens in one step for the model to check in one pass. You get exactly the tokens the model would write on its own, sooner: on an M4 Pro about 2.1–2.2× on maths and 1.2–1.5× on code. Chat text drafts poorly, so there it falls back to plain decoding (about 0.7–1.1×). Both files are copied into this browser's storage once. Tools are off for now.</li>
|
| 2686 |
</ul>
|
| 2687 |
</details>
|
| 2688 |
</div>
|
|
|
|
| 2999 |
<option value="gemma4-26b-a4b-ssd">Gemma 4 26B-A4B · SSD-streamed, experimental (~14 GB)</option>
|
| 3000 |
<option value="lfm25-24b-a2b-ssd">LFM2.5 24B-A2B · SSD-streamed, experimental (~13.6 GB)</option>
|
| 3001 |
<option value="flint-40b-a2b-ssd">Flint 40B-A2.5B · SSD-streamed, experimental, coding (~26 GB)</option>
|
| 3002 |
+
<option value="qwen3-4b-flashmap">Qwen3 4B · FlashMap speculative decoding, experimental (~5.2 GB)</option>
|
| 3003 |
</select>
|
| 3004 |
<!-- Model picker (Chunk L layer 2): renders #modelSelect's options grouped by what is
|
| 3005 |
already in this browser; picking sets the select's value and fires its change event. -->
|
|
|
|
| 8597 |
if (loaded != null && total) post({ type: 'progress', data: { status: 'loading', file: `weights to GPU (${loaded} of ${total} tensors)` } });
|
| 8598 |
} else if (ev.status === 'weights') {
|
| 8599 |
if (typeof ev.loaded === 'number' && ev.total > 0) post({ type: 'progress', data: { status: 'progress_total', file: WEIGHTS_LABEL, loaded: ev.loaded, total: ev.total } });
|
| 8600 |
+
} else if (ev.status === 'drafter-download') { // a speculative drafter's own file (Qwen3Dense + FlashMap)
|
| 8601 |
+
post({ type: 'progress', data: { status: 'progress_total', file: 'FlashMap drafter into on-device storage', loaded: ev.loaded, total: ev.total } });
|
| 8602 |
} else if (ev.status === 'init') {
|
| 8603 |
post({ type: 'progress', data: { status: 'initiate', file: 'WebGPU device' } });
|
| 8604 |
}
|
|
|
|
| 9387 |
contextSize: 8192,
|
| 9388 |
genConfig: { max_new_tokens: 4096 },
|
| 9389 |
},
|
| 9390 |
+
// backend:'qwen3dense' β Qwen3Dense (qwen35_dense.js) with FlashMap speculative decoding (flashmap_drafter.js):
|
| 9391 |
+
// Qwen's own Q8_0 GGUF of Qwen3-4B (4.28 GB) plus Shen et al.'s flow-map drafter j-shen/FlashMap-Qwen3-4B (0.95 GB
|
| 9392 |
+
// bf16, arXiv:2610.24365) in WGSL. Greedy output is the target's own; measured on an M4 Pro (2026-10-22, 8 prompts):
|
| 9393 |
+
// maths 2.1-2.2x, code 1.2-1.5x, chat ~0.7-1.1x of plain decoding (it falls back to plain steps where drafting
|
| 9394 |
+
// does not pay). Gate: 8/8 vs llama.cpp Metal (plan/2026-10-22-flashmap/results.md).
|
| 9395 |
+
'qwen3-4b-flashmap': {
|
| 9396 |
+
id: 'Qwen/Qwen3-4B-GGUF',
|
| 9397 |
+
label: 'Qwen3 4B Β· FlashMap speculative (experimental)',
|
| 9398 |
+
backend: 'qwen3dense',
|
| 9399 |
+
size: '~5.2 GB',
|
| 9400 |
+
gguf: { file: 'Qwen3-4B-Q8_0.gguf', size: 4280404704 }, // QWEN3_4B_Q8 in qwen35_dense.js (the drafter adds 0.95 GB)
|
| 9401 |
+
type: 'causal',
|
| 9402 |
+
multimodal: false,
|
| 9403 |
+
agentCapable: false,
|
| 9404 |
+
thinking: true,
|
| 9405 |
+
contextSize: 4096,
|
| 9406 |
+
genConfig: { max_new_tokens: 2048 },
|
| 9407 |
+
},
|
| 9408 |
'minicpm5-2b-gguf': {
|
| 9409 |
id: 'https://huggingface.co/openbmb/MiniCPM5-2B-GGUF/resolve/main/MiniCPM5-2B-Q4_K_M.gguf',
|
| 9410 |
label: 'MiniCPM5 2B Β· GGUF',
|
|
|
|
| 17722 |
'gemma4moe-ssd': { engine: 'gemma4_moe_ssd.js', exportName: 'Gemma4MoeSsd', label: 'Gemma 4 26B-A4B (Q4_0)' },
|
| 17723 |
'lfm2moe-ssd': { engine: 'lfm2_moe_ssd.js', exportName: 'Lfm2MoeSsd', label: 'LFM2.5 24B-A2B (Q4_0)' },
|
| 17724 |
'flint-ssd': { engine: 'flint_moe_ssd.js', exportName: 'FlintMoeSsd', label: 'Flint 40B-A2.5B (Q4_1)' },
|
| 17725 |
+
'qwen3dense': { engine: 'qwen35_dense.js', exportName: 'Qwen3Dense', label: 'Qwen3 4B (Q8_0) + FlashMap' },
|
| 17726 |
'hyv3-ssd': { engine: 'hy_v3_moe_ssd.js', exportName: 'HyV3MoeSsd', label: 'Hy-Memory 30B-A3B (Q8_0)' }, // the memory writer only (WRITER_MODELS)
|
| 17727 |
'qwen35dense': { engine: 'qwen35_dense.js', exportName: 'Qwen35Dense', label: 'Underdog Saluki 27B (IQ2-mix)' },
|
| 17728 |
})[backend] || null;
|
|
|
|
| 19337 |
const ENGINE_STORES = {
|
| 19338 |
'transformers': ['transformers-cache'], 'wllama': ['wllama'], 'lfm2-webgpu': ['gguf-v1'],
|
| 19339 |
'bonsai2-webgpu': ['gguf-cache-v1'], 'gemma4-webgpu': ['safetensors-cache-v1', 'bonsai-pipeline-v1'],
|
| 19340 |
+
'qwen35moe-ssd': ['on disk (OPFS)'], 'minicpmv-ssd': ['on disk (OPFS)'], 'gemma4moe-ssd': ['on disk (OPFS)'], 'lfm2moe-ssd': ['on disk (OPFS)'], 'flint-ssd': ['on disk (OPFS)'], 'qwen3dense': ['on disk (OPFS)'], 'qwen35dense': ['on disk (OPFS)'],
|
| 19341 |
};
|
| 19342 |
function parseSizeBytes(size) {
|
| 19343 |
const m = /([\d.]+)\s*(GB|MB)/i.exec(String(size || ''));
|
|
|
|
| 23954 |
return new DiskformerError(fallback, `${what} failed: ${(e && e.message) || e}`, { cause: e });
|
| 23955 |
}
|
| 23956 |
</script>
|
| 23957 |
+
<script type="text/plain" data-module="flashmap_drafter.js">/* flashmap_drafter.js β FlashMap speculative decoding (Shen et al., arXiv:2610.24365) for Qwen3Dense: the flow-map
|
| 23958 |
+
* drafter j-shen/FlashMap-Qwen3-4B (475M parameters, bf16, Apache-2.0) in WGSL, and the speculative generate loop.
|
| 23959 |
+
* Output is the target's own greedy decoding; the drafter only proposes.
|
| 23960 |
+
*
|
| 23961 |
+
* The math follows the drafter's modeling_flashmap.py (rev 4e8d6a3). The drafter drafts the 16 tokens after the last
|
| 23962 |
+
* verified token (the anchor) in one network evaluation, a jump of a flow map from noise (s = 0) to tokens (t = 1):
|
| 23963 |
+
* context: the target's hidden states at layers 1, 9, 17, 25, 33 (concatenated, 12800) of every verified position β
|
| 23964 |
+
* fc (12800 β 2048) β RMSNorm, then each DiT layer's cross-attention keys (k_proj, per-head RMSNorm, RoPE) and
|
| 23965 |
+
* values (v_proj), cached; the anchor's slot gets anchor_norm(anchor_proj(E[anchor])) the same way.
|
| 23966 |
+
* input: zero noise, so every slot starts from state_norm(uniform_embeds) (uniform_embeds = the mean row of the target's
|
| 23967 |
+
* token table E); x_embed (2560 β 2048). The noise term noise_norm(zΒ·Lα΅) of a random draft is zero at z = 0, which
|
| 23968 |
+
* also removes the Cholesky factor of Eα΅E (zero noise matched random noise on acceptance: plan/2026-10-22-flashmap).
|
| 23969 |
+
* 6 DiT layers, each: x += gateΒ·self_attn(LN(x)Β·(1+scale)+shift) over the 16 slots (no mask); x += gateΒ·cross_attn(β¦)
|
| 23970 |
+
* over the cached context; x += gateΒ·MLP(β¦) (fc1, exact GELU, fc2). scale/shift/gate per norm come from the time
|
| 23971 |
+
* conditioning of the jump (0 β 1), constant, computed once here in JS. Attention: 16 heads Γ 128, RMSNorm on q and
|
| 23972 |
+
* k, RoPE on interleaved channel pairs (2i, 2i+1), base 10000, softmax(qkα΅/β128).
|
| 23973 |
+
* output: final LNΒ·(1+scale)+shift, head (2048 οΏ½οΏ½οΏ½ 2560), then the target's lm_head (here: the tied Q8_0 token table
|
| 23974 |
+
* through Qwen3Dense's GEMM) and an argmax per slot.
|
| 23975 |
+
* Weights stay bf16 on the GPU (the safetensors bytes as they are) and are decoded in the kernels; activations are f32.
|
| 23976 |
+
*
|
| 23977 |
+
* const target = await Qwen3Dense.load(null, { captureLayers: FLASHMAP_QWEN3_4B_LAYERS, β¦ });
|
| 23978 |
+
* const drafter = await FlashMapDrafter.load(target, { url });
|
| 23979 |
+
* for await (const step of specGenerate(target, drafter, promptIds, { maxNewTokens })) β¦
|
| 23980 |
+
*/
|
| 23981 |
+
|
| 23982 |
+
import { STORAGE, COPY_SRC, COPY_DST, MAP_READ, UNIFORM } from './qwen3_moe_ssd.js';
|
| 23983 |
+
import { OpfsWriter, OpfsReaderPool, opfsFileSize, readOpfsText, writeOpfsText } from './opfs-reader.js';
|
| 23984 |
+
import { MM_TILE, MM_SG_TOKENS, MVT_ROWS, gpuRowBytes } from './iq_quants.js';
|
| 23985 |
+
|
| 23986 |
+
export const FLASHMAP_QWEN3_4B = {
|
| 23987 |
+
repo: 'j-shen/FlashMap-Qwen3-4B',
|
| 23988 |
+
file: 'model.safetensors',
|
| 23989 |
+
revision: '4e8d6a39a84ad108c5b60cb59072eb3f968c98cd',
|
| 23990 |
+
size: 949897866,
|
| 23991 |
+
};
|
| 23992 |
+
export const FLASHMAP_QWEN3_4B_LAYERS = [1, 9, 17, 25, 33];
|
| 23993 |
+
const CFG = { dim: 2048, heads: 16, hd: 128, layers: 6, ff: 8192, block: 16, targetDim: 2560, cond: 256, timeDim: 256,
|
| 23994 |
+
timeScale: 1000, theta: 10000, lnEps: 1e-6, rmsEps: 1e-6, qkEps: 1e-5 };
|
| 23995 |
+
|
| 23996 |
+
// ββ WGSL ββββββββββββββββββββββββββββββββββββββββββββββββββββββββββββββββββββ
|
| 23997 |
+
const BF = 'fn bf(w: u32, hi: bool) -> f32 { return bitcast<f32>(select(w << 16u, w & 0xffff0000u, hi)); }';
|
| 23998 |
+
const KS = {
|
| 23999 |
+
// y[tΒ·M + m] = Ξ£_k x[tΒ·N + k]Β·W[m][k] (+ b[m]); W bf16 [M][N]. 16 rows Γ 16 tokens per workgroup, 32-wide k tiles
|
| 24000 |
+
// through shared memory. Dispatch (ceil(M / 16), ceil(T / 16)).
|
| 24001 |
+
gemmBf16: `
|
| 24002 |
+
struct P { M: u32, N: u32, T: u32, bias: u32 }
|
| 24003 |
+
@group(0) @binding(0) var<storage, read> x: array<f32>;
|
| 24004 |
+
@group(0) @binding(1) var<storage, read> w: array<u32>;
|
| 24005 |
+
@group(0) @binding(2) var<storage, read> b: array<u32>;
|
| 24006 |
+
@group(0) @binding(3) var<storage, read_write> y: array<f32>;
|
| 24007 |
+
@group(0) @binding(4) var<uniform> p: P;
|
| 24008 |
+
${BF}
|
| 24009 |
+
var<workgroup> ws: array<f32, 512>;
|
| 24010 |
+
var<workgroup> xs: array<f32, 512>;
|
| 24011 |
+
@compute @workgroup_size(256) fn main(@builtin(workgroup_id) wg: vec3<u32>, @builtin(local_invocation_id) l: vec3<u32>) {
|
| 24012 |
+
let m0 = wg.x * 16u; let t0 = wg.y * 16u; let r = l.x / 16u; let c = l.x % 16u;
|
| 24013 |
+
var acc = 0.0;
|
| 24014 |
+
for (var k0 = 0u; k0 < p.N; k0 += 32u) {
|
| 24015 |
+
for (var i = l.x; i < 512u; i += 256u) {
|
| 24016 |
+
let rr = i / 32u; let kk = i % 32u; let m = m0 + rr; let t = t0 + rr; let k = k0 + kk;
|
| 24017 |
+
var wv = 0.0;
|
| 24018 |
+
if (m < p.M && k < p.N) { let e = m * p.N + k; wv = bf(w[e >> 1u], (e & 1u) == 1u); }
|
| 24019 |
+
ws[i] = wv;
|
| 24020 |
+
var xv = 0.0;
|
| 24021 |
+
if (t < p.T && k < p.N) { xv = x[t * p.N + k]; }
|
| 24022 |
+
xs[i] = xv;
|
| 24023 |
+
}
|
| 24024 |
+
workgroupBarrier();
|
| 24025 |
+
for (var kk = 0u; kk < 32u; kk++) { acc += ws[r * 32u + kk] * xs[c * 32u + kk]; }
|
| 24026 |
+
workgroupBarrier();
|
| 24027 |
+
}
|
| 24028 |
+
let m = m0 + r; let t = t0 + c;
|
| 24029 |
+
if (m < p.M && t < p.T) {
|
| 24030 |
+
var v = acc;
|
| 24031 |
+
if (p.bias == 1u) { v += bf(b[m >> 1u], (m & 1u) == 1u); }
|
| 24032 |
+
y[t * p.M + m] = v;
|
| 24033 |
+
}
|
| 24034 |
+
}`,
|
| 24035 |
+
// The same product for a few tokens (T β€ TT: the draft's 16 slots, a verify's accepted rows, the anchor): one row per
|
| 24036 |
+
// lane, 256 rows per workgroup, each weight read once; per 256-wide k block the T activation rows go through shared
|
| 24037 |
+
// memory (broadcast reads). The token loop has a constant trip count so the sums stay in registers. N must be a
|
| 24038 |
+
// multiple of 256. Dispatch (ceil(M / 256)).
|
| 24039 |
+
gemvT17: gemvT(17),
|
| 24040 |
+
gemvT1: gemvT(1),
|
| 24041 |
+
// y = x Β· (w Β· rsqrt(mean(xΒ²) + eps)) per row of D; one workgroup per row.
|
| 24042 |
+
rmsRows: `
|
| 24043 |
+
struct P { D: u32, eps: f32 }
|
| 24044 |
+
@group(0) @binding(0) var<storage, read> x: array<f32>;
|
| 24045 |
+
@group(0) @binding(1) var<storage, read> w: array<u32>;
|
| 24046 |
+
@group(0) @binding(2) var<storage, read_write> y: array<f32>;
|
| 24047 |
+
@group(0) @binding(3) var<uniform> p: P;
|
| 24048 |
+
${BF}
|
| 24049 |
+
var<workgroup> red: array<f32, 256>;
|
| 24050 |
+
@compute @workgroup_size(256) fn main(@builtin(workgroup_id) wg: vec3<u32>, @builtin(local_invocation_id) l: vec3<u32>) {
|
| 24051 |
+
let base = wg.x * p.D; let t = l.x;
|
| 24052 |
+
var a = 0.0;
|
| 24053 |
+
for (var i = t; i < p.D; i += 256u) { let v = x[base + i]; a += v * v; }
|
| 24054 |
+
red[t] = a; workgroupBarrier();
|
| 24055 |
+
for (var s = 128u; s > 0u; s >>= 1u) { if (t < s) { red[t] += red[t + s]; } workgroupBarrier(); }
|
| 24056 |
+
let r = 1.0 / sqrt(red[0] / f32(p.D) + p.eps);
|
| 24057 |
+
for (var i = t; i < p.D; i += 256u) { y[base + i] = x[base + i] * (bf(w[i >> 1u], (i & 1u) == 1u) * r); }
|
| 24058 |
+
}`,
|
| 24059 |
+
// y = LayerNorm(x) (no affine) Β· (1 + scale) + shift per row of D; scale/shift at md[off], md[off + D] (`mod` is a
|
| 24060 |
+
// reserved word in WGSL).
|
| 24061 |
+
lnMod: `
|
| 24062 |
+
struct P { D: u32, off: u32, eps: f32, pad: u32 }
|
| 24063 |
+
@group(0) @binding(0) var<storage, read> x: array<f32>;
|
| 24064 |
+
@group(0) @binding(1) var<storage, read> md: array<f32>;
|
| 24065 |
+
@group(0) @binding(2) var<storage, read_write> y: array<f32>;
|
| 24066 |
+
@group(0) @binding(3) var<uniform> p: P;
|
| 24067 |
+
var<workgroup> red: array<f32, 256>;
|
| 24068 |
+
@compute @workgroup_size(256) fn main(@builtin(workgroup_id) wg: vec3<u32>, @builtin(local_invocation_id) l: vec3<u32>) {
|
| 24069 |
+
let base = wg.x * p.D; let t = l.x;
|
| 24070 |
+
var a = 0.0;
|
| 24071 |
+
for (var i = t; i < p.D; i += 256u) { a += x[base + i]; }
|
| 24072 |
+
red[t] = a; workgroupBarrier();
|
| 24073 |
+
for (var s = 128u; s > 0u; s >>= 1u) { if (t < s) { red[t] += red[t + s]; } workgroupBarrier(); }
|
| 24074 |
+
let mu = red[0] / f32(p.D); workgroupBarrier();
|
| 24075 |
+
var q = 0.0;
|
| 24076 |
+
for (var i = t; i < p.D; i += 256u) { let d = x[base + i] - mu; q += d * d; }
|
| 24077 |
+
red[t] = q; workgroupBarrier();
|
| 24078 |
+
for (var s = 128u; s > 0u; s >>= 1u) { if (t < s) { red[t] += red[t + s]; } workgroupBarrier(); }
|
| 24079 |
+
let r = 1.0 / sqrt(red[0] / f32(p.D) + p.eps);
|
| 24080 |
+
for (var i = t; i < p.D; i += 256u) { y[base + i] = (x[base + i] - mu) * r * (1.0 + md[p.off + i]) + md[p.off + p.D + i]; }
|
| 24081 |
+
}`,
|
| 24082 |
+
// In place, per (head wg.x, row wg.y): RMSNorm over hd with weight w, then RoPE on interleaved pairs (2i, 2i+1) with
|
| 24083 |
+
// the row's [cos hd/2 | sin hd/2] in rope.
|
| 24084 |
+
headNormRope: `
|
| 24085 |
+
struct P { hd: u32, row: u32, eps: f32, pad: u32 }
|
| 24086 |
+
@group(0) @binding(0) var<storage, read_write> x: array<f32>;
|
| 24087 |
+
@group(0) @binding(1) var<storage, read> w: array<u32>;
|
| 24088 |
+
@group(0) @binding(2) var<storage, read> rope: array<f32>;
|
| 24089 |
+
@group(0) @binding(3) var<uniform> p: P;
|
| 24090 |
+
${BF}
|
| 24091 |
+
var<workgroup> red: array<f32, 64>;
|
| 24092 |
+
@compute @workgroup_size(64) fn main(@builtin(workgroup_id) wg: vec3<u32>, @builtin(local_invocation_id) l: vec3<u32>) {
|
| 24093 |
+
let base = wg.y * p.row + wg.x * p.hd; let t = l.x; let half = p.hd / 2u; let rb = wg.y * p.hd;
|
| 24094 |
+
var a = 0.0;
|
| 24095 |
+
for (var i = t; i < p.hd; i += 64u) { let v = x[base + i]; a += v * v; }
|
| 24096 |
+
red[t] = a; workgroupBarrier();
|
| 24097 |
+
for (var s = 32u; s > 0u; s >>= 1u) { if (t < s) { red[t] += red[t + s]; } workgroupBarrier(); }
|
| 24098 |
+
let r = 1.0 / sqrt(red[0] / f32(p.hd) + p.eps);
|
| 24099 |
+
workgroupBarrier();
|
| 24100 |
+
for (var i = t; i < half; i += 64u) {
|
| 24101 |
+
let a0 = x[base + 2u * i] * (bf(w[i], false) * r);
|
| 24102 |
+
let a1 = x[base + 2u * i + 1u] * (bf(w[i], true) * r);
|
| 24103 |
+
let c = rope[rb + i]; let sn = rope[rb + half + i];
|
| 24104 |
+
x[base + 2u * i] = a0 * c - a1 * sn;
|
| 24105 |
+
x[base + 2u * i + 1u] = a1 * c + a0 * sn;
|
| 24106 |
+
}
|
| 24107 |
+
}`,
|
| 24108 |
+
// Attention without a mask: query row wg.y, head wg.x, over keys [0, L) of k/v (f32 rows of D); scores in shared
|
| 24109 |
+
// memory (L β€ 4096).
|
| 24110 |
+
attnF32: attn('f32', ''),
|
| 24111 |
+
// The same over the f16 context cache.
|
| 24112 |
+
attnF16: attn('f16', 'enable f16;'),
|
| 24113 |
+
// cache[(start + t)Β·D + i] = f16(x[tΒ·D + i]) for both k and v. Dispatch (ceil(D / 256), T).
|
| 24114 |
+
kvWrite: `enable f16;
|
| 24115 |
+
struct P { D: u32, start: u32, T: u32, pad: u32 }
|
| 24116 |
+
@group(0) @binding(0) var<storage, read> k: array<f32>;
|
| 24117 |
+
@group(0) @binding(1) var<storage, read> v: array<f32>;
|
| 24118 |
+
@group(0) @binding(2) var<storage, read_write> kc: array<f16>;
|
| 24119 |
+
@group(0) @binding(3) var<storage, read_write> vc: array<f16>;
|
| 24120 |
+
@group(0) @binding(4) var<uniform> p: P;
|
| 24121 |
+
@compute @workgroup_size(256) fn main(@builtin(global_invocation_id) g: vec3<u32>) {
|
| 24122 |
+
let i = g.x; let t = g.y; if (i >= p.D || t >= p.T) { return; }
|
| 24123 |
+
kc[(p.start + t) * p.D + i] = f16(k[t * p.D + i]); vc[(p.start + t) * p.D + i] = f16(v[t * p.D + i]);
|
| 24124 |
+
}`,
|
| 24125 |
+
// x = x Β· Ξ¦(x) (exact GELU; erf from AbramowitzβStegun 7.1.26, |error| β€ 1.5e-7).
|
| 24126 |
+
gelu: `
|
| 24127 |
+
struct P { n: u32 }
|
| 24128 |
+
@group(0) @binding(0) var<storage, read_write> x: array<f32>;
|
| 24129 |
+
@group(0) @binding(1) var<uniform> p: P;
|
| 24130 |
+
fn erf_(z: f32) -> f32 {
|
| 24131 |
+
let s = sign(z); let a = abs(z); let t = 1.0 / (1.0 + 0.3275911 * a);
|
| 24132 |
+
let y = 1.0 - (((((1.061405429 * t - 1.453152027) * t) + 1.421413741) * t - 0.284496736) * t + 0.254829592) * t * exp(-a * a);
|
| 24133 |
+
return s * y;
|
| 24134 |
+
}
|
| 24135 |
+
@compute @workgroup_size(256) fn main(@builtin(global_invocation_id) g: vec3<u32>, @builtin(num_workgroups) ng: vec3<u32>) {
|
| 24136 |
+
let i = g.y * ng.x * 256u + g.x; if (i >= p.n) { return; }
|
| 24137 |
+
let v = x[i]; x[i] = 0.5 * v * (1.0 + erf_(v * 0.70710678118654752));
|
| 24138 |
+
}`,
|
| 24139 |
+
// x[tΒ·D + i] += md[off + i] Β· y[tΒ·D + i] (the gated residual). Dispatch (ceil(D / 256), T).
|
| 24140 |
+
gateAdd: `
|
| 24141 |
+
struct P { D: u32, off: u32, T: u32, pad: u32 }
|
| 24142 |
+
@group(0) @binding(0) var<storage, read_write> x: array<f32>;
|
| 24143 |
+
@group(0) @binding(1) var<storage, read> y: array<f32>;
|
| 24144 |
+
@group(0) @binding(2) var<storage, read> md: array<f32>;
|
| 24145 |
+
@group(0) @binding(3) var<uniform> p: P;
|
| 24146 |
+
@compute @workgroup_size(256) fn main(@builtin(global_invocation_id) g: vec3<u32>) {
|
| 24147 |
+
let i = g.x; let t = g.y; if (i >= p.D || t >= p.T) { return; }
|
| 24148 |
+
x[t * p.D + i] += md[p.off + i] * y[t * p.D + i];
|
| 24149 |
+
}`,
|
| 24150 |
+
// x[tΒ·D + i] = r[i] for t < T (every slot starts from the same input at zero noise).
|
| 24151 |
+
broadcast: `
|
| 24152 |
+
struct P { D: u32, T: u32 }
|
| 24153 |
+
@group(0) @binding(0) var<storage, read> r: array<f32>;
|
| 24154 |
+
@group(0) @binding(1) var<storage, read_write> x: array<f32>;
|
| 24155 |
+
@group(0) @binding(2) var<uniform> p: P;
|
| 24156 |
+
@compute @workgroup_size(256) fn main(@builtin(global_invocation_id) g: vec3<u32>) {
|
| 24157 |
+
let i = g.x; let t = g.y; if (i >= p.D || t >= p.T) { return; }
|
| 24158 |
+
x[t * p.D + i] = r[i];
|
| 24159 |
+
}`,
|
| 24160 |
+
// The mean row of the target's Q8_0 token table in Qwen3Dense's padded layout (36-byte blocks: f16 d, 2 pad bytes,
|
| 24161 |
+
// 32 int8): one thread per column; uniform_embeds = (1/V)Β·1α΅E.
|
| 24162 |
+
colMeanQ8: `
|
| 24163 |
+
struct P { V: u32, H: u32, rowWords: u32, pad: u32 }
|
| 24164 |
+
@group(0) @binding(0) var<storage, read> e: array<u32>;
|
| 24165 |
+
@group(0) @binding(1) var<storage, read_write> y: array<f32>;
|
| 24166 |
+
@group(0) @binding(2) var<uniform> p: P;
|
| 24167 |
+
@compute @workgroup_size(256) fn main(@builtin(global_invocation_id) g: vec3<u32>) {
|
| 24168 |
+
let i = g.x; if (i >= p.H) { return; }
|
| 24169 |
+
let blk = i / 32u; let j = i % 32u; var acc = 0.0;
|
| 24170 |
+
for (var r = 0u; r < p.V; r++) {
|
| 24171 |
+
let w0 = r * p.rowWords + blk * 9u;
|
| 24172 |
+
let d = unpack2x16float(e[w0]).x;
|
| 24173 |
+
let q = (e[w0 + 1u + j / 4u] >> ((j % 4u) * 8u)) & 0xffu;
|
| 24174 |
+
acc += d * f32(select(i32(q), i32(q) - 256, q >= 128u));
|
| 24175 |
+
}
|
| 24176 |
+
y[i] = acc / f32(p.V);
|
| 24177 |
+
}`,
|
| 24178 |
+
};
|
| 24179 |
+
function gemvT(TT) {
|
| 24180 |
+
return `
|
| 24181 |
+
struct P { M: u32, N: u32, T: u32, bias: u32 }
|
| 24182 |
+
@group(0) @binding(0) var<storage, read> x: array<f32>;
|
| 24183 |
+
@group(0) @binding(1) var<storage, read> w: array<u32>;
|
| 24184 |
+
@group(0) @binding(2) var<storage, read> b: array<u32>;
|
| 24185 |
+
@group(0) @binding(3) var<storage, read_write> y: array<f32>;
|
| 24186 |
+
@group(0) @binding(4) var<uniform> p: P;
|
| 24187 |
+
${BF}
|
| 24188 |
+
const TT = ${TT}u;
|
| 24189 |
+
var<workgroup> xs: array<vec4<f32>, ${TT * 64}>;
|
| 24190 |
+
@compute @workgroup_size(256) fn main(@builtin(workgroup_id) wg: vec3<u32>, @builtin(local_invocation_id) l: vec3<u32>) {
|
| 24191 |
+
let t = l.x; let row = wg.x * 256u + t; let rr = min(row, p.M - 1u); let T = min(p.T, TT); let rowW = p.N / 2u;
|
| 24192 |
+
var acc: array<f32, ${TT}>;
|
| 24193 |
+
for (var kb = 0u; kb < p.N; kb += 256u) {
|
| 24194 |
+
workgroupBarrier();
|
| 24195 |
+
for (var i = t; i < TT * 64u; i += 256u) {
|
| 24196 |
+
let tk = i / 64u; let o = tk * p.N + kb + (i % 64u) * 4u;
|
| 24197 |
+
xs[i] = select(vec4<f32>(0.0), vec4<f32>(x[o], x[o + 1u], x[o + 2u], x[o + 3u]), tk < T);
|
| 24198 |
+
}
|
| 24199 |
+
workgroupBarrier();
|
| 24200 |
+
let base = rr * rowW + kb / 2u;
|
| 24201 |
+
for (var j = 0u; j < 32u; j++) {
|
| 24202 |
+
let u0 = w[base + 4u * j]; let u1 = w[base + 4u * j + 1u]; let u2 = w[base + 4u * j + 2u]; let u3 = w[base + 4u * j + 3u];
|
| 24203 |
+
let w0 = vec4<f32>(bf(u0, false), bf(u0, true), bf(u1, false), bf(u1, true));
|
| 24204 |
+
let w1 = vec4<f32>(bf(u2, false), bf(u2, true), bf(u3, false), bf(u3, true));
|
| 24205 |
+
for (var tk = 0u; tk < TT; tk++) { acc[tk] += dot(w0, xs[tk * 64u + 2u * j]) + dot(w1, xs[tk * 64u + 2u * j + 1u]); }
|
| 24206 |
+
}
|
| 24207 |
+
}
|
| 24208 |
+
if (row < p.M) {
|
| 24209 |
+
var bv = 0.0;
|
| 24210 |
+
if (p.bias == 1u) { bv = bf(b[row >> 1u], (row & 1u) == 1u); }
|
| 24211 |
+
for (var tk = 0u; tk < TT; tk++) { if (tk < T) { y[tk * p.M + row] = acc[tk] + bv; } }
|
| 24212 |
+
}
|
| 24213 |
+
}`;
|
| 24214 |
+
}
|
| 24215 |
+
function attn(kt, en) {
|
| 24216 |
+
return `${en}
|
| 24217 |
+
struct P { heads: u32, hd: u32, L: u32, scale: f32 }
|
| 24218 |
+
@group(0) @binding(0) var<storage, read> q: array<f32>;
|
| 24219 |
+
@group(0) @binding(1) var<storage, read> k: array<${kt}>;
|
| 24220 |
+
@group(0) @binding(2) var<storage, read> v: array<${kt}>;
|
| 24221 |
+
@group(0) @binding(3) var<storage, read_write> o: array<f32>;
|
| 24222 |
+
@group(0) @binding(4) var<uniform> p: P;
|
| 24223 |
+
var<workgroup> sc: array<f32, 4096>;
|
| 24224 |
+
var<workgroup> red: array<f32, 256>;
|
| 24225 |
+
var<workgroup> qs: array<f32, 128>;
|
| 24226 |
+
@compute @workgroup_size(256) fn main(@builtin(workgroup_id) wg: vec3<u32>, @builtin(local_invocation_id) l: vec3<u32>) {
|
| 24227 |
+
let h = wg.x; let row = wg.y; let t = l.x; let D = p.heads * p.hd; let qb = row * D + h * p.hd;
|
| 24228 |
+
if (t < p.hd) { qs[t] = q[qb + t]; }
|
| 24229 |
+
workgroupBarrier();
|
| 24230 |
+
var mx = -3.4e38;
|
| 24231 |
+
for (var j = t; j < p.L; j += 256u) {
|
| 24232 |
+
var s = 0.0; let kb = j * D + h * p.hd;
|
| 24233 |
+
for (var d = 0u; d < p.hd; d++) { s += qs[d] * f32(k[kb + d]); }
|
| 24234 |
+
s = s * p.scale; sc[j] = s; mx = max(mx, s);
|
| 24235 |
+
}
|
| 24236 |
+
red[t] = mx; workgroupBarrier();
|
| 24237 |
+
for (var s = 128u; s > 0u; s >>= 1u) { if (t < s) { red[t] = max(red[t], red[t + s]); } workgroupBarrier(); }
|
| 24238 |
+
let m = red[0]; workgroupBarrier();
|
| 24239 |
+
var sum = 0.0;
|
| 24240 |
+
for (var j = t; j < p.L; j += 256u) { let e = exp(sc[j] - m); sc[j] = e; sum += e; }
|
| 24241 |
+
red[t] = sum; workgroupBarrier();
|
| 24242 |
+
for (var s = 128u; s > 0u; s >>= 1u) { if (t < s) { red[t] += red[t + s]; } workgroupBarrier(); }
|
| 24243 |
+
let inv = 1.0 / red[0];
|
| 24244 |
+
if (t < p.hd) {
|
| 24245 |
+
var acc = 0.0;
|
| 24246 |
+
for (var j = 0u; j < p.L; j++) { acc += sc[j] * f32(v[j * D + h * p.hd + t]); }
|
| 24247 |
+
o[qb + t] = acc * inv;
|
| 24248 |
+
}
|
| 24249 |
+
}`;
|
| 24250 |
+
}
|
| 24251 |
+
|
| 24252 |
+
// ββ safetensors βββββββββββββββββββββββββββββββββββββββββββββββββββββββββββββ
|
| 24253 |
+
async function readSafetensors(url, fetchFn) {
|
| 24254 |
+
const r0 = await fetchFn(url, { headers: { Range: 'bytes=0-7' } });
|
| 24255 |
+
if (!r0.ok && r0.status !== 206) throw new Error(`drafter: HTTP ${r0.status}`);
|
| 24256 |
+
const n = Number(new DataView(await r0.arrayBuffer()).getBigUint64(0, true));
|
| 24257 |
+
const r1 = await fetchFn(url, { headers: { Range: `bytes=8-${8 + n - 1}` } });
|
| 24258 |
+
const header = JSON.parse(new TextDecoder().decode(await r1.arrayBuffer()));
|
| 24259 |
+
delete header.__metadata__;
|
| 24260 |
+
for (const t of Object.values(header)) if (t.dtype !== 'BF16') throw new Error(`drafter: ${t.dtype} tensor, expected BF16`);
|
| 24261 |
+
return { header, dataStart: 8 + n };
|
| 24262 |
+
}
|
| 24263 |
+
// The drafter file in OPFS (localmind-ssd/flashmap-<repo>/model.safetensors): streamed in once (a partial copy resumes
|
| 24264 |
+
// from its size), then read from there. A marker file (<file>.ok, the pinned revision) is written only after a full,
|
| 24265 |
+
// size-checked copy; without it the copy is fetched again from byte 0. Returns a fetch-like function serving Range
|
| 24266 |
+
// requests from the copy.
|
| 24267 |
+
async function opfsCopy(url, fetchFn, src, onProgress) {
|
| 24268 |
+
const path = `localmind-ssd/flashmap-${src.repo.split('/').pop().toLowerCase()}/${src.file}`, ok = `${path}.ok`;
|
| 24269 |
+
let have = (await opfsFileSize(path)) || 0;
|
| 24270 |
+
const done = have === src.size && (await readOpfsText(ok).catch(() => null)) === src.revision;
|
| 24271 |
+
if (have >= src.size && !done) have = 0;
|
| 24272 |
+
if (!done) {
|
| 24273 |
+
if (have > src.size) have = 0;
|
| 24274 |
+
const w = await OpfsWriter.open(path, { truncate: have === 0 });
|
| 24275 |
+
const r = await fetchFn(url, have ? { headers: { Range: `bytes=${have}-` } } : {});
|
| 24276 |
+
if (!r.ok && r.status !== 206) throw new Error(`drafter download: HTTP ${r.status}`);
|
| 24277 |
+
if (have && r.status !== 206) have = 0; // the server ignored the range: start over
|
| 24278 |
+
const reader = r.body.getReader();
|
| 24279 |
+
let at = have;
|
| 24280 |
+
for (;;) {
|
| 24281 |
+
const { done, value } = await reader.read();
|
| 24282 |
+
if (done) break;
|
| 24283 |
+
const n = value.byteLength; // write() transfers the buffer, so take the length first
|
| 24284 |
+
await w.write(value, at); at += n;
|
| 24285 |
+
onProgress({ status: 'drafter-download', loaded: at, total: src.size });
|
| 24286 |
+
}
|
| 24287 |
+
await w.close();
|
| 24288 |
+
if (at !== src.size) throw new Error(`drafter download stopped at ${at} of ${src.size} bytes; it resumes on the next load`);
|
| 24289 |
+
await writeOpfsText(ok, src.revision);
|
| 24290 |
+
}
|
| 24291 |
+
const pool = await OpfsReaderPool.open(path, { workers: 2 });
|
| 24292 |
+
const read = async (u, init = {}) => {
|
| 24293 |
+
const m = /bytes=(\d+)-(\d+)/.exec((init.headers && init.headers.Range) || '');
|
| 24294 |
+
const start = m ? Number(m[1]) : 0, end = m ? Number(m[2]) : src.size - 1;
|
| 24295 |
+
const got = await pool.read(start, end - start + 1);
|
| 24296 |
+
if (got.got !== end - start + 1) throw new Error(`drafter: short OPFS read at ${start}`);
|
| 24297 |
+
return { ok: true, status: 206, arrayBuffer: async () => got.buf.slice(0, got.got) };
|
| 24298 |
+
};
|
| 24299 |
+
read.close = () => pool.close();
|
| 24300 |
+
return read;
|
| 24301 |
+
}
|
| 24302 |
+
const bf16ToF32 = (u16) => { const out = new Float32Array(u16.length), u = new Uint32Array(out.buffer); for (let i = 0; i < u16.length; i++) u[i] = u16[i] << 16; return out; };
|
| 24303 |
+
|
| 24304 |
+
// The time conditioning of the jump s β t (FlashMapDraftModel.time_conditioning) in f32: per DiT layer (self, cross,
|
| 24305 |
+
// MLP) and the final norm, each [scale D | shift D | gate D]. Small linears, computed once on the CPU.
|
| 24306 |
+
function timeConditioning(cpu, s, t) {
|
| 24307 |
+
const lin = (name, x) => {
|
| 24308 |
+
const w = cpu[`${name}.weight`], b = cpu[`${name}.bias`], [M, N] = w.shape, y = new Float32Array(M);
|
| 24309 |
+
for (let m = 0; m < M; m++) { let a = b ? b.data[m] : 0; for (let k = 0; k < N; k++) a += w.data[m * N + k] * x[k]; y[m] = Math.fround(a); }
|
| 24310 |
+
return y;
|
| 24311 |
+
};
|
| 24312 |
+
const silu = (x) => x.map((v) => v / (1 + Math.exp(-v)));
|
| 24313 |
+
const embed = (prefix, time) => {
|
| 24314 |
+
const half = CFG.timeDim / 2, e = new Float32Array(CFG.timeDim);
|
| 24315 |
+
for (let i = 0; i < half; i++) {
|
| 24316 |
+
const f = Math.fround(Math.exp(-Math.log(10000) * i / half)), a = Math.fround(Math.fround(time * CFG.timeScale) * f);
|
| 24317 |
+
e[i] = Math.cos(a); e[half + i] = Math.sin(a);
|
| 24318 |
+
}
|
| 24319 |
+
return lin(`${prefix}.linear_2`, silu(lin(`${prefix}.linear_1`, e)));
|
| 24320 |
+
};
|
| 24321 |
+
const ts = embed('time_embed', s), tj = embed('jump_embed', t), cond = silu(ts.map((v, i) => v + tj[i]));
|
| 24322 |
+
const out = new Float32Array((CFG.layers * 3 + 1) * 3 * CFG.dim);
|
| 24323 |
+
let o = 0;
|
| 24324 |
+
for (let l = 0; l < CFG.layers; l++) for (const n of ['self_attn_adaln', 'cross_attn_adaln', 'mlp_adaln']) { out.set(lin(`layers.${l}.${n}.linear`, cond), o); o += 3 * CFG.dim; }
|
| 24325 |
+
out.set(lin('final_adaln.linear', cond), o);
|
| 24326 |
+
return out;
|
| 24327 |
+
}
|
| 24328 |
+
// RoPE tables for rows at positions pos0 β¦ pos0 + n β 1: [cos hd/2 | sin hd/2] per row (rope_tables: f32 angles).
|
| 24329 |
+
function ropeRows(pos0, n) {
|
| 24330 |
+
const half = CFG.hd / 2, r = new Float32Array(n * CFG.hd);
|
| 24331 |
+
const inv = Array.from({ length: half }, (_, i) => Math.fround(1 / Math.fround(Math.pow(CFG.theta, Math.fround(2 * i / CFG.hd)))));
|
| 24332 |
+
for (let t = 0; t < n; t++) for (let i = 0; i < half; i++) { const a = Math.fround((pos0 + t) * inv[i]); r[t * CFG.hd + i] = Math.cos(a); r[t * CFG.hd + half + i] = Math.sin(a); }
|
| 24333 |
+
return r;
|
| 24334 |
+
}
|
| 24335 |
+
|
| 24336 |
+
// ββ drafter βββββββββββββββββββββββββββββββββββββββββββββββββββββββββββββββββ
|
| 24337 |
+
export class FlashMapDrafter {
|
| 24338 |
+
// target: a loaded Qwen3Dense with opts.captureLayers = FLASHMAP_QWEN3_4B_LAYERS (it shares the device, the token
|
| 24339 |
+
// table, the lm_head GEMM and the feature rows). opts.url: the safetensors file (default: Hugging Face, pinned).
|
| 24340 |
+
// The file is copied into OPFS once (opts.cache === false reads `url` directly, for a local serve).
|
| 24341 |
+
static async load(target, opts = {}) {
|
| 24342 |
+
const src = opts.source || FLASHMAP_QWEN3_4B, onProgress = opts.onProgress || (() => {});
|
| 24343 |
+
const url = opts.url || `https://huggingface.co/${src.repo}/resolve/${src.revision}/${src.file}`;
|
| 24344 |
+
const net = opts.fetch || ((u, i) => fetch(u, i));
|
| 24345 |
+
const fetchFn = opts.cache === false ? net : await opfsCopy(url, net, src, onProgress);
|
| 24346 |
+
try {
|
| 24347 |
+
const { header, dataStart } = await readSafetensors(url, fetchFn);
|
| 24348 |
+
const d = new FlashMapDrafter(target, opts);
|
| 24349 |
+
await d.upload(url, fetchFn, header, dataStart, onProgress);
|
| 24350 |
+
d.init();
|
| 24351 |
+
return d;
|
| 24352 |
+
} finally { if (fetchFn.close) await fetchFn.close(); }
|
| 24353 |
+
}
|
| 24354 |
+
|
| 24355 |
+
constructor(target, opts) {
|
| 24356 |
+
if (!target.capture || target.featWidth !== FLASHMAP_QWEN3_4B_LAYERS.length * CFG.targetDim) throw new Error('FlashMap: load the target with captureLayers = FLASHMAP_QWEN3_4B_LAYERS');
|
| 24357 |
+
this.t = target; this.device = target.device; this.opts = opts;
|
| 24358 |
+
this.maxCtx = Math.min(target.maxCtx + CFG.block + 1, 4096);
|
| 24359 |
+
this.pipelines = {}; this.gpuBytes = 0;
|
| 24360 |
+
}
|
| 24361 |
+
buffer(size, usage) { const b = this.device.createBuffer({ size: Math.max(16, Math.ceil(size / 4) * 4), usage }); this.gpuBytes += b.size; return b; }
|
| 24362 |
+
uniform(words) {
|
| 24363 |
+
const ab = new ArrayBuffer(Math.max(16, Math.ceil(words.length / 4) * 16)), u = new Uint32Array(ab), f = new Float32Array(ab);
|
| 24364 |
+
words.forEach((w, i) => { if (typeof w === 'object') f[i] = w.f; else u[i] = w; });
|
| 24365 |
+
const b = this.device.createBuffer({ size: ab.byteLength, usage: UNIFORM | COPY_DST }); this.device.queue.writeBuffer(b, 0, ab); return b;
|
| 24366 |
+
}
|
| 24367 |
+
pipeline(name) {
|
| 24368 |
+
if (!this.pipelines[name]) {
|
| 24369 |
+
const module = this.device.createShaderModule({ code: KS[name], label: `flashmap ${name}` });
|
| 24370 |
+
this.pipelines[name] = this.device.createComputePipeline({ layout: 'auto', compute: { module, entryPoint: 'main' }, label: `flashmap ${name}` });
|
| 24371 |
+
}
|
| 24372 |
+
return this.pipelines[name];
|
| 24373 |
+
}
|
| 24374 |
+
bind(name, res) {
|
| 24375 |
+
return this.device.createBindGroup({ layout: this.pipeline(name).getBindGroupLayout(0), entries: res.map((r, i) => ({ binding: i, resource: r instanceof GPUBuffer ? { buffer: r } : r })) });
|
| 24376 |
+
}
|
| 24377 |
+
run(pass, name, group, x, y = 1) { pass.setPipeline(this.pipeline(name)); pass.setBindGroup(0, group); pass.dispatchWorkgroups(x, y); }
|
| 24378 |
+
|
| 24379 |
+
// Every tensor to its own GPU buffer, bf16 bytes as they are; the time-conditioning linears and the noise gate also to
|
| 24380 |
+
// the CPU. Reads the data section in ranges of up to 64 MB.
|
| 24381 |
+
async upload(url, fetchFn, header, dataStart, onProgress) {
|
| 24382 |
+
const names = Object.keys(header).sort((a, b) => header[a].data_offsets[0] - header[b].data_offsets[0]);
|
| 24383 |
+
const cpuWanted = (n) => /^(time_embed|jump_embed)\.|adaln\.linear\.|^noise_gate$/.test(n);
|
| 24384 |
+
this.w = {}; this.cpu = {};
|
| 24385 |
+
const total = header[names[names.length - 1]].data_offsets[1];
|
| 24386 |
+
const CH = 64 << 20;
|
| 24387 |
+
let i = 0, done = 0;
|
| 24388 |
+
while (i < names.length) {
|
| 24389 |
+
const start = header[names[i]].data_offsets[0];
|
| 24390 |
+
let j = i, end = start;
|
| 24391 |
+
while (j < names.length && (header[names[j]].data_offsets[1] - start <= CH || j === i)) { end = header[names[j]].data_offsets[1]; j++; }
|
| 24392 |
+
const r = await fetchFn(url, { headers: { Range: `bytes=${dataStart + start}-${dataStart + end - 1}` } });
|
| 24393 |
+
if (!r.ok && r.status !== 206) throw new Error(`drafter: HTTP ${r.status} reading weights`);
|
| 24394 |
+
const buf = new Uint8Array(await r.arrayBuffer());
|
| 24395 |
+
if (buf.byteLength !== end - start) throw new Error(`drafter: short read ${buf.byteLength} of ${end - start}`);
|
| 24396 |
+
for (let k = i; k < j; k++) {
|
| 24397 |
+
const n = names[k], [a, b] = header[n].data_offsets, bytes = buf.subarray(a - start, b - start);
|
| 24398 |
+
const padded = new Uint8Array(Math.ceil(bytes.byteLength / 4) * 4); padded.set(bytes);
|
| 24399 |
+
const g = this.buffer(padded.byteLength, STORAGE | COPY_DST);
|
| 24400 |
+
this.device.queue.writeBuffer(g, 0, padded);
|
| 24401 |
+
this.w[n] = g;
|
| 24402 |
+
if (cpuWanted(n)) this.cpu[n] = { shape: header[n].shape, data: bf16ToF32(new Uint16Array(padded.buffer, 0, bytes.byteLength / 2)) };
|
| 24403 |
+
}
|
| 24404 |
+
done = end; onProgress({ status: 'drafter', loaded: done, total });
|
| 24405 |
+
i = j;
|
| 24406 |
+
}
|
| 24407 |
+
}
|
| 24408 |
+
|
| 24409 |
+
init() {
|
| 24410 |
+
const D = CFG.dim, H = CFG.targetDim, K = CFG.block, F = CFG.ff, t = this.t, dev = this.device;
|
| 24411 |
+
for (const k of Object.keys(KS)) this.pipeline(k); // compile now, not in the first rounds
|
| 24412 |
+
const maxRows = Math.max(K, t.chunkTokens), A = (n) => this.buffer(n * 4, STORAGE | COPY_SRC | COPY_DST);
|
| 24413 |
+
this.mod = this.buffer((CFG.layers * 3 + 1) * 3 * D * 4, STORAGE | COPY_DST | COPY_SRC);
|
| 24414 |
+
dev.queue.writeBuffer(this.mod, 0, timeConditioning(this.cpu, 0, 1));
|
| 24415 |
+
this.zero = this.buffer(16, STORAGE);
|
| 24416 |
+
this.kc = []; this.vc = [];
|
| 24417 |
+
for (let l = 0; l < CFG.layers; l++) { this.kc.push(this.buffer(this.maxCtx * D * 2, STORAGE)); this.vc.push(this.buffer(this.maxCtx * D * 2, STORAGE)); }
|
| 24418 |
+
const a = this.a = {
|
| 24419 |
+
x: A(K * D), h: A(K * D), q: A(K * D), k: A(K * D), v: A(K * D), o: A(K * D), y: A(K * D), u: A(K * F), out: A(K * H),
|
| 24420 |
+
logits: A(K * t.cfg.vocab), am: this.buffer(K * 4, STORAGE | COPY_SRC), rbAm: this.buffer(K * 4, MAP_READ | COPY_DST),
|
| 24421 |
+
ctx: A(maxRows * D), ctxN: A(maxRows * D), ck: A(maxRows * D), cv: A(maxRows * D), ropeC: A(maxRows * CFG.hd), ropeS: A(K * CFG.hd), ropeA: A(CFG.hd),
|
| 24422 |
+
anchorId: this.buffer(16, STORAGE | COPY_DST), anchorE: A(H), anchorC0: A(D), anchorC: A(D), anchorK: A(D), anchorV: A(D),
|
| 24423 |
+
uniform: A(H), inRow: A(H), h0: A(D), qbK: this.buffer(16, UNIFORM | COPY_DST), qbA: this.buffer(16, UNIFORM | COPY_DST),
|
| 24424 |
+
};
|
| 24425 |
+
dev.queue.writeBuffer(a.qbK, 0, new Uint32Array([K, 0, 0, 0]));
|
| 24426 |
+
dev.queue.writeBuffer(a.qbA, 0, new Uint32Array([1, 0, 0, 0]));
|
| 24427 |
+
const W = this.w, lw = (l, n) => W[`layers.${l}.${n}`];
|
| 24428 |
+
// GEMM ops: { group, M, T }, the y grid is ceil(T / 16).
|
| 24429 |
+
const gemm = (x, name, y, M, N, T, bias = true) => {
|
| 24430 |
+
const res = [x, W[`${name}.weight`], bias && W[`${name}.bias`] ? W[`${name}.bias`] : this.zero, y, this.uniform([M, N, T, bias && W[`${name}.bias`] ? 1 : 0])];
|
| 24431 |
+
return { res, M, N, T };
|
| 24432 |
+
};
|
| 24433 |
+
this.gemm = gemm;
|
| 24434 |
+
const modOff = (n, part) => n * 3 * D + part * D; // part 0 scale (lnMod reads scale, shift), 2 gate
|
| 24435 |
+
const u = {
|
| 24436 |
+
rmsD: this.uniform([D, { f: CFG.rmsEps }]), rmsH: this.uniform([H, { f: CFG.rmsEps }]),
|
| 24437 |
+
qkK: this.uniform([CFG.hd, D, { f: CFG.qkEps }, 0]), self: this.uniform([CFG.heads, CFG.hd, K, { f: Math.fround(1 / Math.sqrt(CFG.hd)) }]),
|
| 24438 |
+
gelu: this.uniform([K * F]), bK: this.uniform([D, K]),
|
| 24439 |
+
};
|
| 24440 |
+
this.u = u;
|
| 24441 |
+
// The target's pieces: its token table (anchor embedding, uniform_embeds) and its lm_head GEMM over 16 slots.
|
| 24442 |
+
const emb = t.w['token_embd.weight'].raw, et = t.types['token_embd.weight'], rowBytes = gpuRowBytes(et, H);
|
| 24443 |
+
this.g = {
|
| 24444 |
+
anchorE: t.bindR(`embedB_${et}`, [emb, t.grid, a.anchorId, a.anchorE, t.uniform([H, rowBytes]), a.qbA]),
|
| 24445 |
+
anchorP: gemm(a.anchorE, 'anchor_proj', a.anchorC0, D, H, 1),
|
| 24446 |
+
anchorN: this.bind('rmsRows', [a.anchorC0, W['anchor_norm.weight'], a.anchorC, u.rmsD]),
|
| 24447 |
+
bcast: this.bind('broadcast', [a.h0, a.x, u.bK]),
|
| 24448 |
+
// The 16 slots through the target's head on its verify-block matvec (one read of the table), else its GEMM.
|
| 24449 |
+
lm: t.kernels[`mvT_${et}`]
|
| 24450 |
+
? { k: `mvT_${et}`, g: t.bindR(`mvT_${et}`, [a.out, emb, t.grid, a.logits, t.uniform([t.cfg.vocab, H, rowBytes, 0]), a.qbK]), wg: Math.ceil(t.cfg.vocab / MVT_ROWS), small: true }
|
| 24451 |
+
: { k: `mm_${et}`, g: t.bindR(`mm_${et}`, [a.out, emb, t.grid, a.logits, t.uniform([t.cfg.vocab, H, rowBytes, 0]), a.qbK]), wg: Math.ceil(t.cfg.vocab / MM_TILE) },
|
| 24452 |
+
am: t.bind('argmaxRows', [a.logits, a.am, t.uniform([t.cfg.vocab])]),
|
| 24453 |
+
final: this.bind('lnMod', [a.x, this.mod, a.h, this.uniform([D, modOff(CFG.layers * 3, 0), { f: CFG.lnEps }, 0])]),
|
| 24454 |
+
head: gemm(a.h, 'head', a.out, H, D, K),
|
| 24455 |
+
};
|
| 24456 |
+
this.layers = [];
|
| 24457 |
+
for (let l = 0; l < CFG.layers; l++) {
|
| 24458 |
+
const ln = (n) => this.bind('lnMod', [a.x, this.mod, a.h, this.uniform([D, modOff(3 * l + n, 0), { f: CFG.lnEps }, 0])]);
|
| 24459 |
+
const ga = (n) => this.bind('gateAdd', [a.x, a.y, this.mod, this.uniform([D, modOff(3 * l + n, 2), K, 0])]);
|
| 24460 |
+
const p = (n) => `layers.${l}.${n}`;
|
| 24461 |
+
this.layers.push({
|
| 24462 |
+
lnS: ln(0), sq: gemm(a.h, p('self_attn.q_proj'), a.q, D, D, K, false), sk: gemm(a.h, p('self_attn.k_proj'), a.k, D, D, K, false), sv: gemm(a.h, p('self_attn.v_proj'), a.v, D, D, K, false),
|
| 24463 |
+
sqn: this.bind('headNormRope', [a.q, lw(l, 'self_attn.q_norm.weight'), a.ropeS, u.qkK]), skn: this.bind('headNormRope', [a.k, lw(l, 'self_attn.k_norm.weight'), a.ropeS, u.qkK]),
|
| 24464 |
+
sat: this.bind('attnF32', [a.q, a.k, a.v, a.o, u.self]), so: gemm(a.o, p('self_attn.o_proj'), a.y, D, D, K, false), gaS: ga(0),
|
| 24465 |
+
lnC: ln(1), cq: gemm(a.h, p('cross_attn.q_proj'), a.q, D, D, K, false), cqn: this.bind('headNormRope', [a.q, lw(l, 'cross_attn.q_norm.weight'), a.ropeS, u.qkK]),
|
| 24466 |
+
co: gemm(a.o, p('cross_attn.o_proj'), a.y, D, D, K, false), gaC: ga(1),
|
| 24467 |
+
lnM: ln(2), f1: gemm(a.h, p('mlp.fc1'), a.u, F, D, K), gl: this.bind('gelu', [a.u, u.gelu]), f2: gemm(a.u, p('mlp.fc2'), a.y, D, F, K), gaM: ga(2),
|
| 24468 |
+
// The anchor's context slot.
|
| 24469 |
+
ak: gemm(a.anchorC, p('cross_attn.k_proj'), a.anchorK, D, D, 1, false), av: gemm(a.anchorC, p('cross_attn.v_proj'), a.anchorV, D, D, 1, false),
|
| 24470 |
+
akn: this.bind('headNormRope', [a.anchorK, lw(l, 'cross_attn.k_norm.weight'), a.ropeA, u.qkK]),
|
| 24471 |
+
});
|
| 24472 |
+
}
|
| 24473 |
+
// Constant input of every slot at zero noise: x_embed(state_norm(uniform_embeds)).
|
| 24474 |
+
const enc = dev.createCommandEncoder(), pass = enc.beginComputePass();
|
| 24475 |
+
const rowWords = rowBytes / 4;
|
| 24476 |
+
this.run(pass, 'colMeanQ8', this.bind('colMeanQ8', [emb, a.uniform, this.uniform([t.cfg.vocab, H, rowWords, 0])]), Math.ceil(H / 256));
|
| 24477 |
+
this.run(pass, 'rmsRows', this.bind('rmsRows', [a.uniform, W['state_norm.weight'], a.inRow, u.rmsH]), 1);
|
| 24478 |
+
this.runGemm(pass, gemm(a.inRow, 'x_embed', a.h0, D, H, 1));
|
| 24479 |
+
pass.end(); dev.queue.submit([enc.finish()]);
|
| 24480 |
+
this.length = 0;
|
| 24481 |
+
}
|
| 24482 |
+
// A GEMM op on the kernel its token count suits: gemvT1 / gemvT17 (one read of the weights) for up to 17 tokens, the
|
| 24483 |
+
// tiled GEMM for longer runs (a prompt's context rows). Bind groups are made on first use and kept on the op.
|
| 24484 |
+
runGemm(pass, op) {
|
| 24485 |
+
const T = op.T, k = T <= 1 ? 'gemvT1' : T <= 17 && op.N % 256 === 0 ? 'gemvT17' : 'gemmBf16';
|
| 24486 |
+
const g = (op.groups ||= {})[k] ||= this.bind(k, op.res);
|
| 24487 |
+
pass.setPipeline(this.pipeline(k)); pass.setBindGroup(0, g);
|
| 24488 |
+
if (k === 'gemmBf16') pass.dispatchWorkgroups(Math.ceil(op.M / 16), Math.ceil(T / 16)); else pass.dispatchWorkgroups(Math.ceil(op.M / 256));
|
| 24489 |
+
}
|
| 24490 |
+
reset() { this.length = 0; }
|
| 24491 |
+
|
| 24492 |
+
// Append the target's feature rows 0 β¦ n β 1 (its b.feat after a chunk) as context positions start β¦ start + n β 1.
|
| 24493 |
+
extendContext(n, start) {
|
| 24494 |
+
if (start + n + CFG.block + 1 > this.maxCtx) throw new Error('FlashMap: context full');
|
| 24495 |
+
const D = CFG.dim, a = this.a, t = this.t, dev = this.device;
|
| 24496 |
+
dev.queue.writeBuffer(a.ropeC, 0, ropeRows(start, n));
|
| 24497 |
+
const enc = dev.createCommandEncoder(), pass = enc.beginComputePass();
|
| 24498 |
+
this.runGemm(pass, this.gemm(t.b.feat, 'fc', a.ctx, D, t.featWidth, n, false));
|
| 24499 |
+
this.run(pass, 'rmsRows', this.bind('rmsRows', [a.ctx, this.w['hidden_norm.weight'], a.ctxN, this.u.rmsD]), n);
|
| 24500 |
+
for (let l = 0; l < CFG.layers; l++) {
|
| 24501 |
+
const p = (x) => `layers.${l}.cross_attn.${x}`;
|
| 24502 |
+
this.runGemm(pass, this.gemm(a.ctxN, p('k_proj'), a.ck, D, D, n, false));
|
| 24503 |
+
this.runGemm(pass, this.gemm(a.ctxN, p('v_proj'), a.cv, D, D, n, false));
|
| 24504 |
+
this.run(pass, 'headNormRope', this.bind('headNormRope', [a.ck, this.w[p('k_norm.weight')], a.ropeC, this.u.qkK]), CFG.heads, n);
|
| 24505 |
+
this.run(pass, 'kvWrite', this.bind('kvWrite', [a.ck, a.cv, this.kc[l], this.vc[l], this.uniform([D, start, n, 0])]), Math.ceil(D / 256), n);
|
| 24506 |
+
}
|
| 24507 |
+
pass.end(); dev.queue.submit([enc.finish()]);
|
| 24508 |
+
this.length = start + n;
|
| 24509 |
+
}
|
| 24510 |
+
|
| 24511 |
+
// The 16 tokens after `anchor`, which sits at context position `pos` (= the number of verified tokens before it).
|
| 24512 |
+
async draft(anchor, pos) {
|
| 24513 |
+
const D = CFG.dim, K = CFG.block, a = this.a, t = this.t, dev = this.device, g = this.g;
|
| 24514 |
+
dev.queue.writeBuffer(a.anchorId, 0, new Uint32Array([anchor]));
|
| 24515 |
+
dev.queue.writeBuffer(a.ropeA, 0, ropeRows(pos, 1));
|
| 24516 |
+
dev.queue.writeBuffer(a.ropeS, 0, ropeRows(pos + 1, K));
|
| 24517 |
+
const L = pos + 1, cross = this.uniform([CFG.heads, CFG.hd, L, { f: Math.fround(1 / Math.sqrt(CFG.hd)) }]);
|
| 24518 |
+
const enc = dev.createCommandEncoder(), pass = enc.beginComputePass();
|
| 24519 |
+
// The anchor's context slot at pos.
|
| 24520 |
+
t.dispatch(pass, `embedB_${t.types['token_embd.weight']}`, g.anchorE, Math.ceil(CFG.targetDim / 8 / 256), 1); // the target's kernel
|
| 24521 |
+
this.runGemm(pass, g.anchorP); this.run(pass, 'rmsRows', g.anchorN, 1);
|
| 24522 |
+
for (let l = 0; l < CFG.layers; l++) {
|
| 24523 |
+
const y = this.layers[l];
|
| 24524 |
+
this.runGemm(pass, y.ak); this.runGemm(pass, y.av);
|
| 24525 |
+
this.run(pass, 'headNormRope', y.akn, CFG.heads, 1);
|
| 24526 |
+
this.run(pass, 'kvWrite', this.bind('kvWrite', [a.anchorK, a.anchorV, this.kc[l], this.vc[l], this.uniform([D, pos, 1, 0])]), Math.ceil(D / 256), 1);
|
| 24527 |
+
}
|
| 24528 |
+
// The jump 0 β 1 from zero noise.
|
| 24529 |
+
this.run(pass, 'broadcast', g.bcast, Math.ceil(D / 256), K);
|
| 24530 |
+
for (let l = 0; l < CFG.layers; l++) {
|
| 24531 |
+
const y = this.layers[l];
|
| 24532 |
+
this.run(pass, 'lnMod', y.lnS, K);
|
| 24533 |
+
this.runGemm(pass, y.sq); this.runGemm(pass, y.sk); this.runGemm(pass, y.sv);
|
| 24534 |
+
this.run(pass, 'headNormRope', y.sqn, CFG.heads, K); this.run(pass, 'headNormRope', y.skn, CFG.heads, K);
|
| 24535 |
+
this.run(pass, 'attnF32', y.sat, CFG.heads, K);
|
| 24536 |
+
this.runGemm(pass, y.so); this.run(pass, 'gateAdd', y.gaS, Math.ceil(D / 256), K);
|
| 24537 |
+
this.run(pass, 'lnMod', y.lnC, K);
|
| 24538 |
+
this.runGemm(pass, y.cq); this.run(pass, 'headNormRope', y.cqn, CFG.heads, K);
|
| 24539 |
+
this.run(pass, 'attnF16', this.bind('attnF16', [a.q, this.kc[l], this.vc[l], a.o, cross]), CFG.heads, K);
|
| 24540 |
+
this.runGemm(pass, y.co); this.run(pass, 'gateAdd', y.gaC, Math.ceil(D / 256), K);
|
| 24541 |
+
this.run(pass, 'lnMod', y.lnM, K);
|
| 24542 |
+
this.runGemm(pass, y.f1); this.run(pass, 'gelu', y.gl, Math.ceil(K * CFG.ff / 256));
|
| 24543 |
+
this.runGemm(pass, y.f2); this.run(pass, 'gateAdd', y.gaM, Math.ceil(D / 256), K);
|
| 24544 |
+
}
|
| 24545 |
+
this.run(pass, 'lnMod', g.final, K);
|
| 24546 |
+
this.runGemm(pass, g.head);
|
| 24547 |
+
if (g.lm.small) t.dispatch(pass, g.lm.k, g.lm.g, g.lm.wg);
|
| 24548 |
+
else t.dispatch(pass, g.lm.k, g.lm.g, g.lm.wg, Math.ceil(K / (t.matrixKernels ? MM_SG_TOKENS : MM_TILE)));
|
| 24549 |
+
t.dispatch(pass, 'argmaxRows', g.am, K);
|
| 24550 |
+
pass.end();
|
| 24551 |
+
enc.copyBufferToBuffer(a.am, 0, a.rbAm, 0, K * 4);
|
| 24552 |
+
dev.queue.submit([enc.finish()]);
|
| 24553 |
+
await a.rbAm.mapAsync(1);
|
| 24554 |
+
const out = Array.from(new Uint32Array(a.rbAm.getMappedRange().slice(0)));
|
| 24555 |
+
a.rbAm.unmap();
|
| 24556 |
+
return out;
|
| 24557 |
+
}
|
| 24558 |
+
|
| 24559 |
+
// Debug: the 16 slots' DiT output (before the lm_head) of the last draft.
|
| 24560 |
+
async readOut() {
|
| 24561 |
+
const n = CFG.block * CFG.targetDim * 4, rb = this.device.createBuffer({ size: n, usage: MAP_READ | COPY_DST });
|
| 24562 |
+
const enc = this.device.createCommandEncoder(); enc.copyBufferToBuffer(this.a.out, 0, rb, 0, n); this.device.queue.submit([enc.finish()]);
|
| 24563 |
+
await rb.mapAsync(1); const v = new Float32Array(rb.getMappedRange().slice(0)); rb.unmap(); rb.destroy(); return v;
|
| 24564 |
+
}
|
| 24565 |
+
// Debug: the first n values, and counts of NaN and zero, of named working buffers (this.a) or of this.mod.
|
| 24566 |
+
async peek(names, n = 6) {
|
| 24567 |
+
const out = {};
|
| 24568 |
+
for (const name of names) {
|
| 24569 |
+
const src = name === 'mod' ? this.mod : this.a[name], size = Math.min(src.size, 1 << 20);
|
| 24570 |
+
const rb = this.device.createBuffer({ size, usage: MAP_READ | COPY_DST });
|
| 24571 |
+
const enc = this.device.createCommandEncoder(); enc.copyBufferToBuffer(src, 0, rb, 0, size); this.device.queue.submit([enc.finish()]);
|
| 24572 |
+
await rb.mapAsync(1); const v = new Float32Array(rb.getMappedRange().slice(0)); rb.unmap(); rb.destroy();
|
| 24573 |
+
let nan = 0, zero = 0; for (const x of v) { if (Number.isNaN(x)) nan++; else if (x === 0) zero++; }
|
| 24574 |
+
out[name] = { head: Array.from(v.slice(0, n), (x) => +x.toPrecision(5)), nan, zero, of: v.length };
|
| 24575 |
+
}
|
| 24576 |
+
return out;
|
| 24577 |
+
}
|
| 24578 |
+
destroy() { for (const b of Object.values(this.w || {})) b.destroy(); }
|
| 24579 |
+
}
|
| 24580 |
+
|
| 24581 |
+
// Lossless speculative greedy decoding: prompt `ids` β yields { tokens, accepted, drafted } per round (tokens: the new
|
| 24582 |
+
// ones), or { tokens, plain: true } per plain step. The target must be a Qwen3Dense loaded with captureLayers; the
|
| 24583 |
+
// drafter one from FlashMapDrafter.load.
|
| 24584 |
+
// verify: how many of the 16 drafts a round checks. 'adaptive' (default) checks 7 (an 8-token verify block, about half
|
| 24585 |
+
// the cost of 17) while recent rounds accept fewer than 5, else all 16; a number fixes it.
|
| 24586 |
+
// fallback (default on): when rounds yield fewer tokens than plain steps would in the same time (a slow average under
|
| 24587 |
+
// 85% of round time over plain-step time, both measured as it runs), decode plainly for probeEvery tokens, with the steps still feeding the
|
| 24588 |
+
// drafter's context, then try a round again. Chat text drafts poorly; code and maths draft well.
|
| 24589 |
+
export async function* specGenerate(target, drafter, ids, { maxNewTokens = 256, stopTokenIds = target.stopTokenIds, rounds, verify = 'adaptive', fallback = true, probeEvery = 16 } = {}) {
|
| 24590 |
+
const stops = new Set(stopTokenIds || []);
|
| 24591 |
+
let ema = 4, emaTokens = 5, roundMs = null, stepMs = 30, plainLeft = 0, featRows = 0, featStart = 0, catchUp = false, nRounds = 0;
|
| 24592 |
+
target.reset(); drafter.reset();
|
| 24593 |
+
let next = null;
|
| 24594 |
+
for (let i = 0; i < ids.length; i += target.chunkTokens) {
|
| 24595 |
+
const chunk = ids.slice(i, i + target.chunkTokens), last = i + target.chunkTokens >= ids.length;
|
| 24596 |
+
next = await target.prefillChunk(chunk, last ? 'argmax' : 'none');
|
| 24597 |
+
drafter.extendContext(chunk.length, i);
|
| 24598 |
+
}
|
| 24599 |
+
let produced = 1;
|
| 24600 |
+
yield { tokens: [next], accepted: 0, drafted: 0, prefill: true };
|
| 24601 |
+
if (stops.has(next)) return;
|
| 24602 |
+
while (produced < maxNewTokens) {
|
| 24603 |
+
// Near the drafter's context limit: plain steps to the end (no more drafting).
|
| 24604 |
+
if (target.position + 2 * CFG.block + 2 > drafter.maxCtx) {
|
| 24605 |
+
if (target.position >= target.maxCtx) return;
|
| 24606 |
+
const tok = await target.step(next, 'argmax');
|
| 24607 |
+
produced++; next = tok;
|
| 24608 |
+
yield { tokens: [tok], plain: true };
|
| 24609 |
+
if (stops.has(tok)) return;
|
| 24610 |
+
continue;
|
| 24611 |
+
}
|
| 24612 |
+
if (plainLeft > 0) {
|
| 24613 |
+
// A plain step on `next`, its features into the next free feature row.
|
| 24614 |
+
if (featRows === 0) featStart = target.position;
|
| 24615 |
+
const t0 = performance.now();
|
| 24616 |
+
target.stepFeatRow = featRows;
|
| 24617 |
+
const tok = await target.step(next, 'argmax');
|
| 24618 |
+
target.stepFeatRow = null;
|
| 24619 |
+
stepMs = 0.7 * stepMs + 0.3 * (performance.now() - t0);
|
| 24620 |
+
featRows++; plainLeft--; produced++; next = tok;
|
| 24621 |
+
if (plainLeft === 0 || featRows >= target.chunkTokens || stops.has(tok) || produced >= maxNewTokens) { drafter.extendContext(featRows, featStart); featRows = 0; catchUp = true; }
|
| 24622 |
+
yield { tokens: [tok], plain: true };
|
| 24623 |
+
if (stops.has(tok)) return;
|
| 24624 |
+
continue;
|
| 24625 |
+
}
|
| 24626 |
+
const pos = target.position, anchor = next, K = verify === 'adaptive' ? (ema >= 5 ? CFG.block : 7) : Math.min(CFG.block, verify);
|
| 24627 |
+
const t0 = performance.now();
|
| 24628 |
+
const drafts = (await drafter.draft(anchor, pos)).slice(0, K);
|
| 24629 |
+
const t1 = performance.now();
|
| 24630 |
+
const rows = await target.prefillChunk([anchor, ...drafts], 'rows');
|
| 24631 |
+
const t2 = performance.now();
|
| 24632 |
+
let acc = 0;
|
| 24633 |
+
while (acc < K && drafts[acc] === rows[acc]) acc++;
|
| 24634 |
+
let fresh = [...drafts.slice(0, acc), rows[acc]];
|
| 24635 |
+
const hit = fresh.findIndex((x) => stops.has(x));
|
| 24636 |
+
if (hit >= 0) fresh = fresh.slice(0, hit + 1);
|
| 24637 |
+
ema = 0.6 * ema + 0.4 * acc;
|
| 24638 |
+
emaTokens = 0.75 * emaTokens + 0.25 * (acc + 1); nRounds++;
|
| 24639 |
+
// A round right after a plain stretch also waits for that stretch's context; keep it out of the round time.
|
| 24640 |
+
if (!catchUp) roundMs = roundMs == null ? t2 - t0 : 0.6 * roundMs + 0.4 * (t2 - t0);
|
| 24641 |
+
catchUp = false;
|
| 24642 |
+
drafter.extendContext(acc + 1, pos);
|
| 24643 |
+
target.rewind(pos + acc + 1);
|
| 24644 |
+
next = fresh[fresh.length - 1];
|
| 24645 |
+
produced += fresh.length;
|
| 24646 |
+
yield { tokens: fresh, accepted: acc, drafted: K, draftMs: t1 - t0, verifyMs: t2 - t1 };
|
| 24647 |
+
if (hit >= 0 || (rounds && --rounds <= 0)) return;
|
| 24648 |
+
// Fall back when rounds lose (tokens per round under 85% of break-even, on a slow average over at least 4 rounds),
|
| 24649 |
+
// so chat text drops to plain steps while code and maths near break-even keep drafting; after each plain stretch one
|
| 24650 |
+
// round probes again on the same average.
|
| 24651 |
+
if (fallback && nRounds >= 4 && roundMs != null && emaTokens * stepMs < 0.85 * roundMs) plainLeft = probeEvery;
|
| 24652 |
+
}
|
| 24653 |
+
}
|
| 24654 |
+
</script>
|
| 24655 |
<script type="text/plain" data-module="flint_moe_ssd.js">/* flint_moe_ssd.js β Recursive Flint (GGUF arch `flint`, 39.8B total, ~2.5B active) with its routed experts on disk:
|
| 24656 |
* rung 2g of the SSD-streaming ladder. Flint is a coding-agent model (released 2026-10-21, Apache-2.0). As an engine it
|
| 24657 |
* is the Hy-Memory graph (hy_v3_moe_ssd.js, on the LFM2.5 rung) with attention sinks, a sliding window and no RoPE on
|
|
|
|
| 32961 |
}`;
|
| 32962 |
}
|
| 32963 |
|
| 32964 |
+
// A few tokens at once (a speculative verify block): y[tΒ·M + m] = Ξ£_k x[tΒ·N + k]Β·W[m][k] for T β€ MVT_TOKENS, each
|
| 32965 |
+
// weight read and dequantized once. One row per lane (MVT_ROWS rows per workgroup) with one sum per token; per
|
| 32966 |
+
// 256-wide k block the T activation rows go through shared memory, where every lane reads the same addresses (a
|
| 32967 |
+
// broadcast). The token loop has a constant trip count TT so the sums stay in registers (4 rows per lane spilled them:
|
| 32968 |
+
// 520 ms a block); the cost grows with TT, so short blocks take the TT = 8 build. N must be a multiple of 256.
|
| 32969 |
+
// Dispatch (ceil(M / MVT_ROWS)). Measured on an M4 Pro, Qwen3-4B Q8_0, 17 tokens: 88 ms of matrix work, against 98 ms
|
| 32970 |
+
// for a subgroup-matrix version (64 rows Γ 32 tokens, 128-wide k steps) and 172 ms with the weights staged through
|
| 32971 |
+
// shared memory for coalesced reads (plan/2026-10-22-flashmap/results.md).
|
| 32972 |
+
export const MVT_TOKENS = 17, MVT_ROWS = 256;
|
| 32973 |
+
export function mvTKernel(type, TTn = MVT_TOKENS) {
|
| 32974 |
+
return `
|
| 32975 |
+
struct P { M: u32, N: u32, rowBytes: u32, pad: u32 }
|
| 32976 |
+
struct Q { T: u32, pos0: u32, S: u32, pad: u32 }
|
| 32977 |
+
@group(0) @binding(0) var<storage, read> x: array<f32>;
|
| 32978 |
+
@group(0) @binding(1) var<storage, read> wq: array<u32>;
|
| 32979 |
+
@group(0) @binding(2) var<storage, read> grid: array<u32>;
|
| 32980 |
+
@group(0) @binding(3) var<storage, read_write> y: array<f32>;
|
| 32981 |
+
@group(0) @binding(4) var<uniform> p: P;
|
| 32982 |
+
@group(0) @binding(5) var<uniform> qd: Q;
|
| 32983 |
+
${deqFn(type)}
|
| 32984 |
+
const TT = ${TTn}u;
|
| 32985 |
+
var<workgroup> xs: array<vec4<f32>, ${TTn * 64}>; // [token][k / 4] for one 256-wide k block
|
| 32986 |
+
@compute @workgroup_size(${MVT_ROWS}) fn main(@builtin(workgroup_id) wg: vec3<u32>, @builtin(num_workgroups) ng: vec3<u32>, @builtin(local_invocation_id) l: vec3<u32>) {
|
| 32987 |
+
_ = grid[0];
|
| 32988 |
+
let t = l.x; let row = (wg.y * ng.x + wg.x) * ${MVT_ROWS}u + t; let rr = min(row, p.M - 1u); let T = min(qd.T, TT);
|
| 32989 |
+
var acc: array<f32, ${TTn}>;
|
| 32990 |
+
for (var kb = 0u; kb < p.N; kb += 256u) {
|
| 32991 |
+
workgroupBarrier();
|
| 32992 |
+
for (var i = t; i < TT * 64u; i += ${MVT_ROWS}u) {
|
| 32993 |
+
let tk = i / 64u; let o = tk * p.N + kb + (i % 64u) * 4u;
|
| 32994 |
+
xs[i] = select(vec4<f32>(0.0), vec4<f32>(x[o], x[o + 1u], x[o + 2u], x[o + 3u]), tk < T);
|
| 32995 |
+
}
|
| 32996 |
+
workgroupBarrier();
|
| 32997 |
+
for (var c = 0u; c < 8u; c++) {
|
| 32998 |
+
let cc = kb / 32u + c;
|
| 32999 |
+
let B = rr * p.rowBytes + (cc / ${CPB(type)}u) * ${gpuBlock(type)}u; let s = cc % ${CPB(type)}u;
|
| 33000 |
+
for (var g = 0u; g < 4u; g++) {
|
| 33001 |
+
let v = deq8(B, s, g);
|
| 33002 |
+
let w0 = vec4<f32>(v[0], v[1], v[2], v[3]); let w1 = vec4<f32>(v[4], v[5], v[6], v[7]);
|
| 33003 |
+
let xi = c * 8u + g * 2u;
|
| 33004 |
+
for (var tk = 0u; tk < TT; tk++) { acc[tk] += dot(w0, xs[tk * 64u + xi]) + dot(w1, xs[tk * 64u + xi + 1u]); }
|
| 33005 |
+
}
|
| 33006 |
+
}
|
| 33007 |
+
}
|
| 33008 |
+
if (row < p.M) { for (var tk = 0u; tk < TT; tk++) { if (tk < T) { y[tk * p.M + row] = acc[tk]; } } }
|
| 33009 |
+
}`;
|
| 33010 |
+
}
|
| 33011 |
+
|
| 33012 |
// Token-major batch: y[tΒ·M + m] = Ξ£_k x[tΒ·N + k]Β·W[m][k]. A 256-thread workgroup computes a 64-row Γ 64-token tile.
|
| 33013 |
// Per 32-wide k chunk every thread dequantizes 8 weights (64 rows Γ 4 groups) and loads 8 activations into shared
|
| 33014 |
// memory (k-major f32 tiles, 16 KB in all), then accumulates a 4-row Γ 4-token block: each weight read from
|
|
|
|
| 40973 |
<script type="text/plain" data-module="qwen35_dense.js">/* qwen35_dense.js β dense Qwen3.5-family models (GGUF arch `qwen35`, e.g. Qwen3.8-27B) from an imatrix
|
| 40974 |
* IQ-mix GGUF, in a browser tab. First user: Underdog Saluki 27B 1.0 (ConwayResearch, Apache-2.0), one 7.9 GB
|
| 40975 |
* GGUF mixing eleven llama.cpp block types.
|
| 40976 |
+
* Also plain dense Qwen3 (GGUF arch `qwen3`, Qwen3Dense below; first user Qwen3-4B Q8_0, the FlashMap target): every
|
| 40977 |
+
* layer full attention with an ungated Q, q/k RMSNorm and NeoX RoPE over the whole head, `ffn_norm` before the MLP,
|
| 40978 |
+
* and the output head tied to the token table (llama.cpp src/models/qwen3.cpp).
|
| 40979 |
*
|
| 40980 |
* The trunk is the one qwen35_moe_ssd.js runs (llama.cpp's qwen35 graph): 3 of every 4 layers are Gated DeltaNet,
|
| 40981 |
* every 4th is gated full attention with q/k RMSNorm and partial NeoX RoPE; this module reuses those kernels and
|
|
|
|
| 40992 |
|
| 40993 |
import { Qwen3MoeSsd, STORAGE, COPY_SRC, COPY_DST, MAP_READ, UNIFORM } from './qwen3_moe_ssd.js';
|
| 40994 |
import { QWEN35_KERNELS, QWEN35_BATCH_KERNELS } from './qwen35_moe_ssd.js';
|
| 40995 |
+
import { FlashMapDrafter, FLASHMAP_QWEN3_4B_LAYERS, specGenerate } from './flashmap_drafter.js';
|
| 40996 |
+
import { QTYPE_BY_ID, rowBytes, gpuRowBytes, PADDED, MV_ROWS, MM_TILE, MM_SG_TOKENS, MVT_TOKENS, MVT_ROWS, gridData, repackKernel, mvKernel, mmKernel, mmSgKernel, mvTKernel, embedKernel, embedBKernel } from './iq_quants.js';
|
| 40997 |
|
| 40998 |
export const UNDERDOG_SALUKI_27B = {
|
| 40999 |
repo: 'ConwayResearch/Underdog-Saluki-27B-1.0',
|
|
|
|
| 41003 |
size: 7898369152,
|
| 41004 |
};
|
| 41005 |
const F32 = 0;
|
| 41006 |
+
const NO_SSM = { dConv: 0, dInner: 0, dState: 0, vHeads: 0, kHeads: 0 }; // qwen3: no DeltaNet layers
|
| 41007 |
|
| 41008 |
export function configFromGguf(kv) {
|
| 41009 |
const a = kv['general.architecture'];
|
| 41010 |
+
if (a !== 'qwen35' && a !== 'qwen3') throw new Error(`Qwen35Dense needs a qwen35 or qwen3 GGUF, got ${a}`);
|
| 41011 |
+
const g = (k) => kv[`${a}.${k}`], plain = a === 'qwen3';
|
| 41012 |
return {
|
| 41013 |
arch: a,
|
| 41014 |
layers: g('block_count') - (g('nextn_predict_layers') || 0), hidden: g('embedding_length'), ff: g('feed_forward_length'),
|
| 41015 |
heads: g('attention.head_count'), kvHeads: g('attention.head_count_kv'), headDim: g('attention.key_length'),
|
| 41016 |
+
ropeDims: g('rope.dimension_count') || g('attention.key_length'), ropeTheta: g('rope.freq_base'), eps: g('attention.layer_norm_rms_epsilon'),
|
| 41017 |
+
attnInterval: plain ? 1 : g('full_attention_interval') || 4, contextLength: g('context_length'),
|
| 41018 |
+
// qwen35 attention has a sigmoid gate in the Q projection; qwen3's does not. The MLP's norm is named differently.
|
| 41019 |
+
gatedQ: !plain, ffnNorm: plain ? 'ffn_norm.weight' : 'post_attention_norm.weight',
|
| 41020 |
+
ssm: plain ? null : { dConv: g('ssm.conv_kernel'), dInner: g('ssm.inner_size'), dState: g('ssm.state_size'), vHeads: g('ssm.time_step_rank'), kHeads: g('ssm.group_count') },
|
| 41021 |
vocab: kv['tokenizer.ggml.tokens'].length,
|
| 41022 |
bos: kv['tokenizer.ggml.bos_token_id'], eos: kv['tokenizer.ggml.eos_token_id'],
|
| 41023 |
};
|
|
|
|
| 41067 |
@compute @workgroup_size(256) fn main(@builtin(global_invocation_id) gi: vec3<u32>) {
|
| 41068 |
let i = gi.x; if (i >= p.F) { return; }
|
| 41069 |
let g = gu[i]; act[i] = (g / (1.0 + exp(-g))) * gu[p.F + i];
|
| 41070 |
+
}`,
|
| 41071 |
+
// Speculative decoding support (opts.captureLayers): a layer's T output rows into columns [off, off + H) of the
|
| 41072 |
+
// feature rows (width W) a drafter reads. Dispatch (ceil(H / 256), T).
|
| 41073 |
+
copyCols: `
|
| 41074 |
+
struct P { H: u32, W: u32, off: u32, pad: u32 }
|
| 41075 |
+
struct Q { T: u32, pos0: u32, S: u32, pad: u32 }
|
| 41076 |
+
@group(0) @binding(0) var<storage, read> x: array<f32>;
|
| 41077 |
+
@group(0) @binding(1) var<storage, read_write> f: array<f32>;
|
| 41078 |
+
@group(0) @binding(2) var<uniform> p: P;
|
| 41079 |
+
@group(0) @binding(3) var<uniform> qd: Q;
|
| 41080 |
+
@compute @workgroup_size(256) fn main(@builtin(global_invocation_id) g: vec3<u32>) {
|
| 41081 |
+
let i = g.x; let t = g.y; if (i >= p.H || t >= qd.T) { return; }
|
| 41082 |
+
f[t * p.W + p.off + i] = x[t * p.H + i];
|
| 41083 |
+
}`,
|
| 41084 |
+
// The one-token step's version: row r.row of the feature rows (r is a uniform the step writes). Dispatch ceil(H / 256).
|
| 41085 |
+
copyRow: `
|
| 41086 |
+
struct P { H: u32, W: u32, off: u32, pad: u32 }
|
| 41087 |
+
struct R { row: u32, a: u32, b: u32, c: u32 }
|
| 41088 |
+
@group(0) @binding(0) var<storage, read> x: array<f32>;
|
| 41089 |
+
@group(0) @binding(1) var<storage, read_write> f: array<f32>;
|
| 41090 |
+
@group(0) @binding(2) var<uniform> p: P;
|
| 41091 |
+
@group(0) @binding(3) var<uniform> r: R;
|
| 41092 |
+
@compute @workgroup_size(256) fn main(@builtin(global_invocation_id) g: vec3<u32>) {
|
| 41093 |
+
let i = g.x; if (i >= p.H) { return; }
|
| 41094 |
+
f[r.row * p.W + p.off + i] = x[i];
|
| 41095 |
+
}`,
|
| 41096 |
+
// The argmax of each of T rows of n logits (lowest index on ties). One workgroup per row.
|
| 41097 |
+
argmaxRows: `
|
| 41098 |
+
struct P { n: u32 }
|
| 41099 |
+
@group(0) @binding(0) var<storage, read> l: array<f32>;
|
| 41100 |
+
@group(0) @binding(1) var<storage, read_write> out: array<u32>;
|
| 41101 |
+
@group(0) @binding(2) var<uniform> p: P;
|
| 41102 |
+
var<workgroup> bv: array<f32, 256>;
|
| 41103 |
+
var<workgroup> bi: array<u32, 256>;
|
| 41104 |
+
@compute @workgroup_size(256) fn main(@builtin(workgroup_id) wg: vec3<u32>, @builtin(local_invocation_id) lid: vec3<u32>) {
|
| 41105 |
+
let r = wg.x; let t = lid.x; let base = r * p.n;
|
| 41106 |
+
var v = -3.4e38; var ix = 0u;
|
| 41107 |
+
for (var i = t; i < p.n; i += 256u) { let x = l[base + i]; if (x > v) { v = x; ix = i; } }
|
| 41108 |
+
bv[t] = v; bi[t] = ix; workgroupBarrier();
|
| 41109 |
+
for (var s = 128u; s > 0u; s >>= 1u) {
|
| 41110 |
+
if (t < s) { let o = bv[t + s]; if (o > bv[t] || (o == bv[t] && bi[t + s] < bi[t])) { bv[t] = o; bi[t] = bi[t + s]; } }
|
| 41111 |
+
workgroupBarrier();
|
| 41112 |
+
}
|
| 41113 |
+
if (t == 0u) { out[r] = bi[0]; }
|
| 41114 |
}`,
|
| 41115 |
};
|
| 41116 |
+
// Rows of logits a verify chunk can return at once (want = 'rows').
|
| 41117 |
+
const ROWS_MAX = 32;
|
| 41118 |
|
| 41119 |
// Prefill kernels that replace qwen35_moe_ssd.js's batched DeltaNet step and attention (same bindings, same math):
|
| 41120 |
// gdnStepR β the gated delta rule over T tokens with each head's 128Γ128 state in registers (256 threads, 64
|
|
|
|
| 41249 |
constructor(device, manifest, gguf, opts) {
|
| 41250 |
super(device, manifest, gguf, opts);
|
| 41251 |
const c = this.cfg;
|
| 41252 |
+
if (c.ssm && c.ssm.dConv !== 4) throw new Error(`gdnConv assumes a 4-tap conv, got ${c.ssm.dConv}`);
|
| 41253 |
+
if (c.ssm && (c.ssm.dState !== 128 || c.ssm.dInner !== c.ssm.vHeads * c.ssm.dState)) throw new Error('gdnStep assumes 128-wide heads');
|
| 41254 |
this.recurrent = Array.from({ length: c.layers }, (_, l) => (l + 1) % c.attnInterval !== 0);
|
| 41255 |
this.types = {};
|
| 41256 |
for (const [name, d] of Object.entries(manifest.dense.tensors)) this.types[name] = d.ggml === F32 ? null : QTYPE_BY_ID[d.ggml];
|
| 41257 |
+
if (!this.types['output.weight']) this.types['output.weight'] = this.types['token_embd.weight']; // tied head (qwen3)
|
| 41258 |
this.prefetch = false;
|
| 41259 |
// Prompt chunks of up to 256 tokens. Without flash attention the batched scores take heads Γ chunk Γ maxCtx
|
| 41260 |
// floats; then the chunk shrinks to keep them near 96 MB.
|
|
|
|
| 41280 |
const embedType = this.types['token_embd.weight'];
|
| 41281 |
for (const t of new Set(Object.values(this.types).filter(Boolean))) {
|
| 41282 |
k[`mv_${t}`] = mvKernel(t); k[`mm_${t}`] = this.matrixKernels ? mmSgKernel(t) : mmKernel(t);
|
| 41283 |
+
if (this.opts.captureLayers) { k[`mvT_${t}`] = mvTKernel(t); k[`mvT8_${t}`] = mvTKernel(t, 8); } // verify blocks
|
| 41284 |
if (PADDED.has(t)) k[`repack_${t}`] = repackKernel(t);
|
| 41285 |
}
|
| 41286 |
k[`embed_${embedType}`] = embedKernel(embedType);
|
|
|
|
| 41331 |
let pending = [];
|
| 41332 |
const flush = async () => { await dev.queue.onSubmittedWorkDone(); for (const b of pending) { this.gpuBytes.dense -= b.size; b.destroy(); } pending = []; };
|
| 41333 |
for (const [name, t] of Object.entries(this.types)) {
|
| 41334 |
+
if (!t || !PADDED.has(t) || !this.manifest.dense.tensors[name]) continue; // the tied head's alias has no tensor
|
| 41335 |
const dims = this.manifest.dense.tensors[name].dims, rows = dims.reduce((a, b) => a * b, 1) / dims[0];
|
| 41336 |
const words = gpuRowBytes(t, dims[0]) * rows / 4, src = this.w[name].raw;
|
| 41337 |
const dst = this.buffer(words * 4, STORAGE | COPY_DST, 'dense');
|
|
|
|
| 41354 |
}
|
| 41355 |
|
| 41356 |
kvBytes() {
|
| 41357 |
+
const c = this.cfg, s = c.ssm || NO_SSM, convDim = 2 * s.kHeads * s.dState + s.dInner;
|
| 41358 |
return this.recurrent.reduce((sum, r) => sum + (r ? 3 * convDim * 4 + s.vHeads * s.dState * s.dState * 4 : 2 * 2 * this.maxCtx * c.kvHeads * c.headDim), 0);
|
| 41359 |
}
|
| 41360 |
|
|
|
|
| 41368 |
mm(name, x, y, M, N) {
|
| 41369 |
const t = this.types[name], w = this.w[name];
|
| 41370 |
const u = this.uniform([M, N, gpuRowBytes(t, N), 0]);
|
| 41371 |
+
const op = { k: `mm_${t}`, g: this.bindR(`mm_${t}`, [x, w.raw, this.grid, y, u, this.qb]), wg: Math.ceil(M / MM_TILE) };
|
| 41372 |
+
// Speculative verify blocks (T β€ MVT_TOKENS) read each weight once instead: the same bindings, the matvec kernel.
|
| 41373 |
+
if (this.kernels[`mvT_${t}`] && N % 256 === 0) {
|
| 41374 |
+
const v = (k) => ({ k, g: this.bindR(k, [x, w.raw, this.grid, y, u, this.qb]), wg: Math.ceil(M / MVT_ROWS) });
|
| 41375 |
+
op.small = v(`mvT_${t}`); op.small8 = v(`mvT8_${t}`);
|
| 41376 |
+
}
|
| 41377 |
+
return op;
|
| 41378 |
}
|
| 41379 |
|
| 41380 |
initBuffers() {
|
| 41381 |
+
const c = this.cfg, s = c.ssm || NO_SSM;
|
| 41382 |
+
if (!this.w['output.weight']) this.w['output.weight'] = this.w['token_embd.weight']; // tied head (qwen3)
|
| 41383 |
const H = c.hidden, hd = c.headDim, QN = c.heads * hd, kvn = c.kvHeads * hd, F = c.ff;
|
| 41384 |
const keyDim = s.kHeads * s.dState, convDim = 2 * keyDim + s.dInner;
|
| 41385 |
if ((F * 4) % 256) throw new Error('gate/up halves must be 256-byte aligned');
|
|
|
|
| 41407 |
const U = (w) => this.uniform(w), eps = { f: c.eps };
|
| 41408 |
const u = {
|
| 41409 |
nH: U([H]), rmsH: U([H, eps]),
|
| 41410 |
+
ropeQ: U([hd, c.gatedQ ? 2 * hd : hd, c.ropeDims, c.gatedQ ? 1 : 0, eps]), ropeK: U([hd, hd, c.ropeDims, 0, eps]), kvn: U([kvn]),
|
| 41411 |
att: U([c.heads, c.kvHeads, hd, { f: 1 / Math.sqrt(hd) }]), attOut: U([c.heads, c.kvHeads, hd, 0]), gate: U([QN]),
|
| 41412 |
conv: U([convDim]), step: U([s.kHeads, s.dState, keyDim, 0, eps, { f: 1 / Math.sqrt(s.dState) }]), ng: U([s.dState, eps]),
|
| 41413 |
silu: U([F]), am: U([c.vocab]),
|
|
|
|
| 41438 |
});
|
| 41439 |
} else {
|
| 41440 |
Object.assign(g, {
|
| 41441 |
+
q: this.mv(n('attn_q.weight'), a.xn, a.qg, c.gatedQ ? 2 * QN : QN, H),
|
| 41442 |
k: this.mv(n('attn_k.weight'), a.xn, a.k, kvn, H),
|
| 41443 |
v: this.mv(n('attn_v.weight'), a.xn, a.v, kvn, H),
|
| 41444 |
ropeQ: this.bind('qkNormRopeP', [a.qg, raw(n('attn_q_norm.weight')), a.rope, a.q, a.gate, u.ropeQ]),
|
|
|
|
| 41447 |
score: this.bind('attnScore', [a.q, this.kc[l], a.sc, this.tok, u.att]),
|
| 41448 |
soft: this.bind('softmax', [a.sc, this.tok]),
|
| 41449 |
attOut: this.bind('attnOut', [a.sc, this.vc[l], a.att, this.tok, u.attOut]),
|
| 41450 |
+
gate: c.gatedQ ? this.bind('sigmoidMul', [a.att, a.gate, u.gate]) : null,
|
| 41451 |
o: this.mv(n('attn_output.weight'), a.att, a.o, H, QN),
|
| 41452 |
});
|
| 41453 |
}
|
| 41454 |
Object.assign(g, {
|
| 41455 |
addO: this.bind('addInPlace', [a.x, a.o, u.nH]),
|
| 41456 |
+
rmsF: this.bind('rmsnorm', [a.x, raw(n(c.ffnNorm)), a.xn, u.rmsH]),
|
| 41457 |
up: this.mv(n('ffn_up.weight'), a.xn, half(1), F, H),
|
| 41458 |
gt: this.mv(n('ffn_gate.weight'), a.xn, half(0), F, H),
|
| 41459 |
silu: this.bind('siluMul', [a.gu, a.act, u.silu]),
|
|
|
|
| 41468 |
// ββ one token βββββββββββββββββββββββββββββββββββββββββββββββββββββββββββββ
|
| 41469 |
run(pass, op) { this.dispatch(pass, op.k, op.g, op.wg); }
|
| 41470 |
encodeLayer(pass, l, seqLen) {
|
| 41471 |
+
const c = this.cfg, s = c.ssm || NO_SSM, g = this.layers[l], d = (nm, gr, x) => this.dispatch(pass, nm, gr, x);
|
| 41472 |
const H = c.hidden, QN = c.heads * c.headDim, kvn = c.kvHeads * c.headDim, convDim = 2 * s.kHeads * s.dState + s.dInner;
|
| 41473 |
d('rmsnorm', g.rmsA, 1);
|
| 41474 |
if (g.recur) {
|
|
|
|
| 41484 |
d('attnScore', g.score, Math.ceil(c.heads * seqLen / 256));
|
| 41485 |
d('softmax', g.soft, c.heads);
|
| 41486 |
d('attnOut', g.attOut, Math.ceil(QN / 256));
|
| 41487 |
+
if (g.gate) d('sigmoidMul', g.gate, Math.ceil(QN / 256));
|
| 41488 |
this.run(pass, g.o);
|
| 41489 |
}
|
| 41490 |
d('addInPlace', g.addO, Math.ceil(H / 256));
|
|
|
|
| 41493 |
d('siluMul', g.silu, Math.ceil(c.ff / 256));
|
| 41494 |
this.run(pass, g.down);
|
| 41495 |
d('addInPlace', g.addD, Math.ceil(H / 256));
|
| 41496 |
+
const c1 = this.stepFeatRow != null && this.capture1 && this.capture1.get(l);
|
| 41497 |
+
if (c1) d('copyRow', c1, Math.ceil(H / 256));
|
| 41498 |
}
|
| 41499 |
encodeHead(pass, want) {
|
| 41500 |
this.dispatch(pass, 'rmsnorm', this.g.rmsOut, 1);
|
|
|
|
| 41508 |
if (this.position >= this.maxCtx) throw this.contextFull();
|
| 41509 |
const tStart = performance.now(), pos = this.position;
|
| 41510 |
this.writeTokenUniforms(token, pos);
|
| 41511 |
+
if (this.stepFeatRow != null && this.featRowU) this.device.queue.writeBuffer(this.featRowU, 0, new Uint32Array([this.stepFeatRow, 0, 0, 0]));
|
| 41512 |
const enc = dev.createCommandEncoder(), pass = enc.beginComputePass();
|
| 41513 |
this.dispatch(pass, `embed_${this.types['token_embd.weight']}`, this.g.embed, Math.ceil(c.hidden / 8 / 256));
|
| 41514 |
for (let l = 0; l < c.layers; l++) this.encodeLayer(pass, l, pos + 1);
|
|
|
|
| 41545 |
|
| 41546 |
// ββ batched prefill: chunks of T prompt tokens, token-major βββββββββββββββ
|
| 41547 |
initBatch() {
|
| 41548 |
+
const c = this.cfg, s = c.ssm || NO_SSM, B = this.chunkTokens, a = this.a;
|
| 41549 |
const H = c.hidden, hd = c.headDim, QN = c.heads * hd, kvn = c.kvHeads * hd, F = c.ff;
|
| 41550 |
const keyDim = s.kHeads * s.dState, convDim = 2 * keyDim + s.dInner;
|
| 41551 |
const A = (n) => this.buffer(n * 4, STORAGE | COPY_SRC | COPY_DST, 'batch');
|
|
|
|
| 41558 |
const U = (w) => this.uniform(w), eps = { f: c.eps }, qb = this.qb, W = this.w;
|
| 41559 |
const u = {
|
| 41560 |
nH: U([H]), rmsH: U([H, eps]),
|
| 41561 |
+
ropeQ: U(c.gatedQ ? [hd, 2 * hd, 2 * QN, QN, c.ropeDims, 1, eps] : [hd, hd, QN, QN, c.ropeDims, 0, eps]), ropeK: U([hd, hd, kvn, kvn, c.ropeDims, 0, eps]), kvn: U([kvn]),
|
| 41562 |
att: U([c.heads, c.kvHeads, hd, { f: 1 / Math.sqrt(hd) }]), attOut: U([c.heads, c.kvHeads, hd, 0]), gate: U([QN]),
|
| 41563 |
conv: U([convDim]), step: U([s.kHeads, s.dState, keyDim, convDim, s.vHeads, 0, eps, { f: 1 / Math.sqrt(s.dState) }]),
|
| 41564 |
ng: U([s.dState, s.dInner, eps]), ff: U([F]),
|
| 41565 |
};
|
| 41566 |
const raw = (name) => W[name].raw, et = this.types['token_embd.weight'];
|
| 41567 |
this.gB = { embed: this.bind(`embedB_${et}`, [W['token_embd.weight'].raw, this.grid, b.ids, b.x, U([H, gpuRowBytes(et, H)]), qb]) };
|
| 41568 |
+
// Speculative decoding: the chosen layers' output rows, side by side (B Γ layersΒ·H), for a drafter; and a head over
|
| 41569 |
+
// up to ROWS_MAX rows with one argmax per row, for verifying a drafted block in one pass.
|
| 41570 |
+
const cap = this.opts.captureLayers || [];
|
| 41571 |
+
if (cap.length) {
|
| 41572 |
+
this.featWidth = cap.length * H;
|
| 41573 |
+
b.feat = A(B * this.featWidth);
|
| 41574 |
+
this.capture = new Map(cap.map((L, j) => [L, this.bind('copyCols', [b.x, b.feat, U([H, this.featWidth, j * H, 0]), qb])]));
|
| 41575 |
+
// One-token steps with this.stepFeatRow set write their features to that row (a drafter's plain stretch).
|
| 41576 |
+
this.featRowU = this.buffer(16, UNIFORM | COPY_DST, 'batch');
|
| 41577 |
+
this.capture1 = new Map(cap.map((L, j) => [L, this.bind('copyRow', [a.x, b.feat, U([H, this.featWidth, j * H, 0]), this.featRowU])]));
|
| 41578 |
+
b.xnR = A(ROWS_MAX * H); b.logitsR = A(ROWS_MAX * c.vocab);
|
| 41579 |
+
b.amR = this.buffer(ROWS_MAX * 4, STORAGE | COPY_SRC, 'batch');
|
| 41580 |
+
this.rbRows = this.buffer(ROWS_MAX * 4, MAP_READ | COPY_DST, 'batch');
|
| 41581 |
+
this.gB.rmsOutR = this.bind('rmsnormB', [b.x, raw('output_norm.weight'), b.xnR, u.rmsH]);
|
| 41582 |
+
this.gB.lmR = this.mm('output.weight', b.xnR, b.logitsR, c.vocab, H);
|
| 41583 |
+
this.gB.amR = this.bind('argmaxRows', [b.logitsR, b.amR, U([c.vocab])]);
|
| 41584 |
+
}
|
| 41585 |
for (let l = 0; l < c.layers; l++) {
|
| 41586 |
const n = (t) => `blk.${l}.${t}`, g = { rmsA: this.bind('rmsnormB', [b.x, raw(n('attn_norm.weight')), b.xn, u.rmsH]) };
|
| 41587 |
if (this.recurrent[l]) {
|
|
|
|
| 41595 |
});
|
| 41596 |
} else {
|
| 41597 |
Object.assign(g, {
|
| 41598 |
+
q: this.mm(n('attn_q.weight'), b.xn, b.qg, c.gatedQ ? 2 * QN : QN, H), k: this.mm(n('attn_k.weight'), b.xn, b.k, kvn, H), v: this.mm(n('attn_v.weight'), b.xn, b.v, kvn, H),
|
| 41599 |
ropeQ: this.bind('qkNormRopePB', [b.qg, raw(n('attn_q_norm.weight')), b.rope, b.q, b.gate, u.ropeQ]),
|
| 41600 |
ropeK: this.bind('qkNormRopePB', [b.k, raw(n('attn_k_norm.weight')), b.rope, b.kr, b.dummy, u.ropeK]),
|
| 41601 |
kv: this.bind('kvStoreB', [b.kr, b.v, this.kc[l], this.vc[l], u.kvn, qb]),
|
|
|
|
| 41604 |
soft: this.bind('softmaxB', [b.sc, qb]),
|
| 41605 |
attOut: this.bind('attnOutB', [b.sc, this.vc[l], b.att, u.attOut, qb]),
|
| 41606 |
}),
|
| 41607 |
+
gate: c.gatedQ ? this.bind('sigmoidMulB', [b.att, b.gate, u.gate, qb]) : null,
|
| 41608 |
o: this.mm(n('attn_output.weight'), b.att, b.o, H, QN),
|
| 41609 |
});
|
| 41610 |
}
|
| 41611 |
Object.assign(g, {
|
| 41612 |
addO: this.bind('addInPlaceB', [b.x, b.o, u.nH, qb]),
|
| 41613 |
+
rmsF: this.bind('rmsnormB', [b.x, raw(n(c.ffnNorm)), b.xn, u.rmsH]),
|
| 41614 |
gt: this.mm(n('ffn_gate.weight'), b.xn, b.g, F, H), up: this.mm(n('ffn_up.weight'), b.xn, b.u, F, H),
|
| 41615 |
silu: this.bind('siluMul2B', [b.g, b.u, b.act, u.ff, qb]),
|
| 41616 |
down: this.mm(n('ffn_down.weight'), b.act, b.o, H, F),
|
|
|
|
| 41621 |
}
|
| 41622 |
|
| 41623 |
encodeLayerB(pass, l, T, S) {
|
| 41624 |
+
const c = this.cfg, s = c.ssm || NO_SSM, g = this.layers[l].B, d = (nm, gr, x, y) => this.dispatch(pass, nm, gr, x, y);
|
| 41625 |
const H = c.hidden, QN = c.heads * c.headDim, kvn = c.kvHeads * c.headDim, convDim = 2 * s.kHeads * s.dState + s.dInner;
|
| 41626 |
+
const tt = Math.ceil(T / (this.matrixKernels ? MM_SG_TOKENS : MM_TILE));
|
| 41627 |
+
const mm = (op) => {
|
| 41628 |
+
const sm = op.small && (T <= 8 ? op.small8 : T <= MVT_TOKENS ? op.small : null);
|
| 41629 |
+
if (sm) this.dispatch(pass, sm.k, sm.g, sm.wg); else this.dispatch(pass, op.k, op.g, op.wg, tt);
|
| 41630 |
+
};
|
| 41631 |
d('rmsnormB', g.rmsA, T);
|
| 41632 |
if (this.recurrent[l]) {
|
| 41633 |
mm(g.qkv); mm(g.z); mm(g.beta); mm(g.alpha);
|
|
|
|
| 41641 |
d('kvStoreB', g.kv, Math.ceil(kvn / 256), T);
|
| 41642 |
if (g.flash) d('flashAttnB', g.flash, c.kvHeads, Math.ceil(T / 4));
|
| 41643 |
else { d('attnScoreB', g.score, Math.ceil(c.heads * T * S / 256)); d('softmaxB', g.soft, c.heads * T); d('attnOutB', g.attOut, Math.ceil(QN / 256), T); }
|
| 41644 |
+
if (g.gate) d('sigmoidMulB', g.gate, Math.ceil(QN / 256), T);
|
| 41645 |
mm(g.o);
|
| 41646 |
}
|
| 41647 |
d('addInPlaceB', g.addO, Math.ceil(H / 256), T);
|
|
|
|
| 41650 |
d('siluMul2B', g.silu, Math.ceil(c.ff / 256), T);
|
| 41651 |
mm(g.down);
|
| 41652 |
d('addInPlaceB', g.addD, Math.ceil(H / 256), T);
|
| 41653 |
+
const cap = this.capture && this.capture.get(l);
|
| 41654 |
+
if (cap) d('copyCols', cap, Math.ceil(H / 256), T);
|
| 41655 |
}
|
| 41656 |
|
| 41657 |
+
// T = ids.length tokens at this.position through every layer in one submit; want applies to the last token, except
|
| 41658 |
+
// 'rows' (opts.captureLayers set, T β€ ROWS_MAX): the argmax of every row, a Uint32Array of T.
|
| 41659 |
async prefillChunk(ids, want = 'none') {
|
| 41660 |
const c = this.cfg, dev = this.device, H = c.hidden, T = ids.length, pos0 = this.position, S = pos0 + T;
|
| 41661 |
if (T > this.chunkTokens) throw new Error(`chunk of ${T} > ${this.chunkTokens}`);
|
|
|
|
| 41673 |
let pass = enc.beginComputePass();
|
| 41674 |
this.dispatch(pass, `embedB_${this.types['token_embd.weight']}`, this.gB.embed, Math.ceil(H / 8 / 256), T);
|
| 41675 |
for (let l = 0; l < c.layers; l++) this.encodeLayerB(pass, l, T, S);
|
| 41676 |
+
if (want === 'rows') {
|
| 41677 |
+
if (!this.capture || T > ROWS_MAX) throw new Error(`rows: needs captureLayers and at most ${ROWS_MAX} tokens`);
|
| 41678 |
+
const tt = Math.ceil(T / (this.matrixKernels ? MM_SG_TOKENS : MM_TILE)), lm = this.gB.lmR;
|
| 41679 |
+
this.dispatch(pass, 'rmsnormB', this.gB.rmsOutR, T);
|
| 41680 |
+
const sm = lm.small && (T <= 8 ? lm.small8 : T <= MVT_TOKENS ? lm.small : null);
|
| 41681 |
+
if (sm) this.dispatch(pass, sm.k, sm.g, sm.wg); else this.dispatch(pass, lm.k, lm.g, lm.wg, tt);
|
| 41682 |
+
this.dispatch(pass, 'argmaxRows', this.gB.amR, T);
|
| 41683 |
+
}
|
| 41684 |
pass.end();
|
| 41685 |
enc.copyBufferToBuffer(this.b.x, (T - 1) * H * 4, this.a.x, 0, H * 4);
|
| 41686 |
+
if (want === 'rows') enc.copyBufferToBuffer(this.b.amR, 0, this.rbRows, 0, T * 4);
|
| 41687 |
+
else if (want !== 'none') {
|
| 41688 |
pass = enc.beginComputePass();
|
| 41689 |
this.encodeHead(pass, want);
|
| 41690 |
pass.end();
|
|
|
|
| 41694 |
dev.queue.submit([enc.finish()]);
|
| 41695 |
this.position += T;
|
| 41696 |
this.cached.push(...ids);
|
| 41697 |
+
let result;
|
| 41698 |
+
if (want === 'rows') {
|
| 41699 |
+
const tw = performance.now();
|
| 41700 |
+
await this.rbRows.mapAsync(1, 0, T * 4);
|
| 41701 |
+
result = new Uint32Array(this.rbRows.getMappedRange(0, T * 4).slice(0));
|
| 41702 |
+
this.rbRows.unmap();
|
| 41703 |
+
this.counters.gpuWaitMs += performance.now() - tw;
|
| 41704 |
+
} else result = await this.readResult(want);
|
| 41705 |
this.counters.tokens += T;
|
| 41706 |
this.counters.wallMs += performance.now() - tStart;
|
| 41707 |
return result;
|
| 41708 |
}
|
| 41709 |
|
| 41710 |
+
// Back to `pos` tokens (a verify pass that kept only part of its block). Attention layers only: their KV past pos is
|
| 41711 |
+
// overwritten by the next tokens; DeltaNet state cannot rewind.
|
| 41712 |
+
rewind(pos) {
|
| 41713 |
+
if (this.recurrent.some(Boolean)) throw new Error('rewind: this model has recurrent layers');
|
| 41714 |
+
this.position = pos; this.cached.length = pos;
|
| 41715 |
+
}
|
| 41716 |
+
|
| 41717 |
// The recurrent state cannot rewind, so any change before the cached end restarts from token 0.
|
| 41718 |
async prefill(ids, want = 'argmax') {
|
| 41719 |
let common = 0;
|
|
|
|
| 41765 |
async dispose() { await this.retire(); try { this.device.destroy(); } catch (_) {} }
|
| 41766 |
}
|
| 41767 |
|
| 41768 |
+
// Plain dense Qwen3 (arch qwen3) on the same engine. Without a url it loads Qwen's own Q8_0 GGUF of Qwen3-4B, the
|
| 41769 |
+
// target of the FlashMap drafter j-shen/FlashMap-Qwen3-4B (Q8_0 chosen by Chirag 2026-10-22 as closest to the bf16
|
| 41770 |
+
// model the drafter was trained on).
|
| 41771 |
+
export const QWEN3_4B_Q8 = {
|
| 41772 |
+
repo: 'Qwen/Qwen3-4B-GGUF',
|
| 41773 |
+
file: 'Qwen3-4B-Q8_0.gguf',
|
| 41774 |
+
revision: 'bc640142c66e1fdd12af0bd68f40445458f3869b',
|
| 41775 |
+
sha256: '8c2f07f26af9747e41988551106f149b03eb9b5cb6df636027b6bf6278473300',
|
| 41776 |
+
size: 4280404704,
|
| 41777 |
+
};
|
| 41778 |
+
// With FlashMap (default; opts.flashmap === false loads the target alone): the drafter j-shen/FlashMap-Qwen3-4B
|
| 41779 |
+
// (0.95 GB, copied into OPFS once) and speculative greedy decoding in generateTurn (flashmap_drafter.js specGenerate:
|
| 41780 |
+
// the same tokens as plain greedy, faster on code and maths, plain steps where drafting does not pay).
|
| 41781 |
+
export class Qwen3Dense extends Qwen35Dense {
|
| 41782 |
+
static async load(modelId = null, opts = {}) {
|
| 41783 |
+
const source = opts.source || (opts.url ? { repo: null, file: opts.url.split('/').pop(), revision: null } : QWEN3_4B_Q8);
|
| 41784 |
+
const flashmap = opts.flashmap !== false;
|
| 41785 |
+
const m = await super.load(modelId, { ...opts, source, ...(flashmap ? { captureLayers: FLASHMAP_QWEN3_4B_LAYERS } : {}) });
|
| 41786 |
+
if (flashmap && !opts.captureLayers) m.drafter = await FlashMapDrafter.load(m, { fetch: opts.fetch, onProgress: opts.onProgress || (() => {}), url: opts.drafterUrl, cache: opts.drafterCache });
|
| 41787 |
+
return m;
|
| 41788 |
+
}
|
| 41789 |
+
async *generateTurn(messages, opts = {}) {
|
| 41790 |
+
if (!this.drafter) { yield* super.generateTurn(messages, opts); return; }
|
| 41791 |
+
const { maxNewTokens = 512, signal, enableThinking = true } = opts;
|
| 41792 |
+
const ids = this.tokenizer.encode(this.chatPrompt(messages, { enableThinking }));
|
| 41793 |
+
const stops = new Set(this.stopTokenIds), out = [];
|
| 41794 |
+
for await (const r of specGenerate(this, this.drafter, ids, { maxNewTokens })) {
|
| 41795 |
+
for (const tok of r.tokens) {
|
| 41796 |
+
if (stops.has(tok)) return;
|
| 41797 |
+
out.push(tok);
|
| 41798 |
+
yield { text: this.tokenizer.decode(out), token: tok, tokens: out.length };
|
| 41799 |
+
if (out.length >= maxNewTokens) return;
|
| 41800 |
+
}
|
| 41801 |
+
if (signal && signal.aborted) return;
|
| 41802 |
+
}
|
| 41803 |
+
}
|
| 41804 |
+
async dispose() { if (this.drafter) this.drafter.destroy(); await super.dispose(); }
|
| 41805 |
+
}
|
| 41806 |
+
|
| 41807 |
// For kernel tests (scripts/test-qwen35-dense-kernels.mjs).
|
| 41808 |
export { PREFILL_KERNELS as QWEN35_DENSE_PREFILL_KERNELS };
|
| 41809 |
</script>
|