Spaces:
Running
Running
NakliTechie commited on
Commit Β·
53ebc1b
1
Parent(s): 137dd80
Sync from GitHub 2026-10-22T20:59:39Z
Browse files- index.html +933 -2
index.html
CHANGED
|
@@ -32855,7 +32855,8 @@ export async function removeStore(key, { root = ROOT } = {}) { await removeOpfs(
|
|
| 32855 |
<script type="text/plain" data-module="iq_quants.js">/* iq_quants.js β llama.cpp's block quant formats as WGSL, read straight from the GGUF bytes.
|
| 32856 |
*
|
| 32857 |
* Covers the types an imatrix "IQ-mix" GGUF mixes (Underdog Saluki 27B uses all eleven): Q8_0, Q2_K, Q4_K,
|
| 32858 |
-
* IQ1_S, IQ1_M, IQ2_XXS, IQ2_XS, IQ2_S, IQ3_XXS, IQ3_S, IQ4_XS
|
|
|
|
| 32859 |
* on the GPU, one array<u32>: blocks of 4k+2 bytes get two bytes of padding after their f16 scale (repackKernel),
|
| 32860 |
* so every field is word-aligned. `deq8(B, s, g)` returns weights 8g..8g+7 of 32-value chunk s of the block at
|
| 32861 |
* byte offset B, transcribed from ggml-quants.c's dequantize_row_* so the values are the ones llama.cpp computes.
|
|
@@ -32874,6 +32875,9 @@ export const QTYPES = {
|
|
| 32874 |
Q8_0: { id: 8, block: 34, qk: 32 },
|
| 32875 |
Q2_K: { id: 10, block: 84, qk: 256 },
|
| 32876 |
Q4_K: { id: 12, block: 144, qk: 256 },
|
|
|
|
|
|
|
|
|
|
| 32877 |
IQ2_XXS: { id: 16, block: 66, qk: 256 },
|
| 32878 |
IQ2_XS: { id: 17, block: 74, qk: 256 },
|
| 32879 |
IQ3_XXS: { id: 18, block: 98, qk: 256 },
|
|
@@ -33006,6 +33010,27 @@ const DEQ8 = {
|
|
| 33006 |
let qb = B + 16u + 32u * (s >> 1u) + 8u * g; let hi = (s & 1u) * 4u;
|
| 33007 |
let qa = w(qb) >> hi; let qc = w(qb + 4u) >> hi;
|
| 33008 |
for (var j = 0u; j < 4u; j++) { v[j] = dl * f32(byt(qa, j) & 15u) - ml; v[j + 4u] = dl * f32(byt(qc, j) & 15u) - ml; }`,
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
| 33009 |
// dequantize_row_iq2_xxs: per chunk two u32: 4 grid indices, then 4Γ7 sign bits + a 4-bit scale.
|
| 33010 |
IQ2_XXS: `
|
| 33011 |
let d = f16v(B);
|
|
@@ -33087,7 +33112,7 @@ const DEQ8 = {
|
|
| 33087 |
for (var j = 0u; j < 4u; j++) { v[j] = dl * KV4[byt(qa, j) & 15u]; v[j + 4u] = dl * KV4[byt(qc, j) & 15u]; }`,
|
| 33088 |
};
|
| 33089 |
|
| 33090 |
-
function deqFn(type) {
|
| 33091 |
const body = DEQ8[type];
|
| 33092 |
if (!body) throw new Error(`no deq8 for ${type}`);
|
| 33093 |
return `${HELPERS}
|
|
@@ -39991,6 +40016,910 @@ export class Lfm2MoeSsd extends Gemma4MoeSsd {
|
|
| 39991 |
// For kernel tests (scripts): the WGSL this module adds.
|
| 39992 |
export { L2 as LFM2MOE_KERNELS, L2B as LFM2MOE_BATCH_KERNELS };
|
| 39993 |
</script>
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
| 39994 |
<script type="text/plain" data-module="minicpmv_vision.js">/* minicpmv_vision.js β MiniCPM-V 4.7's vision encoder (mmproj GGUF, projector `minicpmv4_7`) on WebGPU.
|
| 39995 |
*
|
| 39996 |
* One image tile in, its token embeddings out (64 Γ 2048 for a 448 Γ 448 tile), ready to splice into the text model.
|
|
@@ -43671,6 +44600,8 @@ export const QWEN3_30B_A3B = {
|
|
| 43671 |
};
|
| 43672 |
const FORMAT = 'localmind-qwen3moe-ssd/1';
|
| 43673 |
const OPFS_ROOT = 'localmind-ssd';
|
|
|
|
|
|
|
| 43674 |
|
| 43675 |
// ββ GGUF ββββββββββββββββββββββββββββββββββββββββββββββββββββββββββββββββββββ
|
| 43676 |
// qwen3moe (rung 2a) and qwen35moe (Qwen3.5/3.6 MoE, rung 2b: qwen35_moe_ssd.js) share the
|
|
|
|
| 32855 |
<script type="text/plain" data-module="iq_quants.js">/* iq_quants.js β llama.cpp's block quant formats as WGSL, read straight from the GGUF bytes.
|
| 32856 |
*
|
| 32857 |
* Covers the types an imatrix "IQ-mix" GGUF mixes (Underdog Saluki 27B uses all eleven): Q8_0, Q2_K, Q4_K,
|
| 32858 |
+
* IQ1_S, IQ1_M, IQ2_XXS, IQ2_XS, IQ2_S, IQ3_XXS, IQ3_S, IQ4_XS; and Q6_K and Q5_0, which a Q4_K_M GGUF adds (Ling 3.0
|
| 32859 |
+
* Flash). Q6_K and Q5_0 are read unpadded, so their rows can sit at any even byte offset (a raw copy of GGUF bytes). A tensor stays in (nearly) its GGUF block layout
|
| 32860 |
* on the GPU, one array<u32>: blocks of 4k+2 bytes get two bytes of padding after their f16 scale (repackKernel),
|
| 32861 |
* so every field is word-aligned. `deq8(B, s, g)` returns weights 8g..8g+7 of 32-value chunk s of the block at
|
| 32862 |
* byte offset B, transcribed from ggml-quants.c's dequantize_row_* so the values are the ones llama.cpp computes.
|
|
|
|
| 32875 |
Q8_0: { id: 8, block: 34, qk: 32 },
|
| 32876 |
Q2_K: { id: 10, block: 84, qk: 256 },
|
| 32877 |
Q4_K: { id: 12, block: 144, qk: 256 },
|
| 32878 |
+
// Q6_K and Q5_0 blocks (210, 22 bytes) are 2 mod 4 and stay unpadded: their deq8 reads bytes and halves only.
|
| 32879 |
+
Q6_K: { id: 14, block: 210, qk: 256 },
|
| 32880 |
+
Q5_0: { id: 6, block: 22, qk: 32 },
|
| 32881 |
IQ2_XXS: { id: 16, block: 66, qk: 256 },
|
| 32882 |
IQ2_XS: { id: 17, block: 74, qk: 256 },
|
| 32883 |
IQ3_XXS: { id: 18, block: 98, qk: 256 },
|
|
|
|
| 33010 |
let qb = B + 16u + 32u * (s >> 1u) + 8u * g; let hi = (s & 1u) * 4u;
|
| 33011 |
let qa = w(qb) >> hi; let qc = w(qb + 4u) >> hi;
|
| 33012 |
for (var j = 0u; j < 4u; j++) { v[j] = dl * f32(byt(qa, j) & 15u) - ml; v[j + 4u] = dl * f32(byt(qc, j) & 15u) - ml; }`,
|
| 33013 |
+
// dequantize_row_q6_K: 2 halves Γ 128 values; value l of quarter k (k = 0..3) of half n takes ql[64n + l + 32(k&1)]
|
| 33014 |
+
// (low nibble for k < 2, high for k β₯ 2), 2 bits of qh[32n + l] at shift 2k, and int8 scale sc[8n + l/16 + 2k]; d last.
|
| 33015 |
+
Q6_K: `
|
| 33016 |
+
let d = f16v(B + 208u);
|
| 33017 |
+
let n = s >> 2u; let k = s & 3u; let l0 = 8u * g;
|
| 33018 |
+
let sc = f32(i32(b8(B + 192u + 8u * n + (l0 >> 4u) + 2u * k) << 24u) >> 24u) * d;
|
| 33019 |
+
let qlB = B + 64u * n + 32u * (k & 1u) + l0; let qhB = B + 128u + 32u * n + l0; let hiNib = (k >> 1u) * 4u;
|
| 33020 |
+
for (var j = 0u; j < 8u; j++) {
|
| 33021 |
+
let q = ((b8(qlB + j) >> hiNib) & 15u) | (((b8(qhB + j) >> (2u * k)) & 3u) << 4u);
|
| 33022 |
+
v[j] = sc * f32(i32(q) - 32);
|
| 33023 |
+
}`,
|
| 33024 |
+
// dequantize_row_q5_0: value i of 32 = (low nibble of qs[i] for i < 16, else high nibble of qs[i-16]) | bit i of qh
|
| 33025 |
+
// as bit 4, minus 16, times d. qh is the u32 at bytes 2..5, read as two halves.
|
| 33026 |
+
Q5_0: `
|
| 33027 |
+
let d = f16v(B);
|
| 33028 |
+
let qh = h16(B + 2u) | (h16(B + 4u) << 16u);
|
| 33029 |
+
for (var j = 0u; j < 8u; j++) {
|
| 33030 |
+
let i = 8u * g + j;
|
| 33031 |
+
let lo = select(b8(B + 6u + i - 16u) >> 4u, b8(B + 6u + i) & 15u, i < 16u);
|
| 33032 |
+
v[j] = d * f32(i32(lo | (((qh >> i) & 1u) << 4u)) - 16);
|
| 33033 |
+
}`,
|
| 33034 |
// dequantize_row_iq2_xxs: per chunk two u32: 4 grid indices, then 4Γ7 sign bits + a 4-bit scale.
|
| 33035 |
IQ2_XXS: `
|
| 33036 |
let d = f16v(B);
|
|
|
|
| 33112 |
for (var j = 0u; j < 4u; j++) { v[j] = dl * KV4[byt(qa, j) & 15u]; v[j + 4u] = dl * KV4[byt(qc, j) & 15u]; }`,
|
| 33113 |
};
|
| 33114 |
|
| 33115 |
+
export function deqFn(type) {
|
| 33116 |
const body = DEQ8[type];
|
| 33117 |
if (!body) throw new Error(`no deq8 for ${type}`);
|
| 33118 |
return `${HELPERS}
|
|
|
|
| 40016 |
// For kernel tests (scripts): the WGSL this module adds.
|
| 40017 |
export { L2 as LFM2MOE_KERNELS, L2B as LFM2MOE_BATCH_KERNELS };
|
| 40018 |
</script>
|
| 40019 |
+
<script type="text/plain" data-module="ling_moe_ssd.js">/* ling_moe_ssd.js β inclusionAI Ling 3.0 Flash (GGUF arch `bailingmoe3`, 124B total, 5.1B active) with its routed
|
| 40020 |
+
* experts on disk: rung 2h of the SSD-streaming ladder. Released 2026-08-02, MIT. The file is inclusionAI's own Q4_K_M
|
| 40021 |
+
* GGUF (inclusionAI/Ling-3.0-flash-GGUF, two parts, 77.0 GB): Q4_K matrices with Q6_K on some, Q5_0 on attn_k_b, F32
|
| 40022 |
+
* norms, routers and small vectors. The engine reads every block type straight from the GGUF bytes (iq_quants.js).
|
| 40023 |
+
*
|
| 40024 |
+
* The math follows llama.cpp src/models/bailingmoe3.cpp and its CPU ops (a37cc461). 42 layers (a 43rd, the MTP head,
|
| 40025 |
+
* is not stored); layers 5, 11, β¦, 41 (head_count_kv = 1) are MLA attention, the other 35 are KDA linear attention:
|
| 40026 |
+
* KDA (Kimi Delta Attention; 32 heads Γ 128): q, k, v projections, each through a 4-tap causal conv and SiLU; q, k
|
| 40027 |
+
* L2-normalised per head (x / β(Ξ£xΒ² + eps)); per-channel decay g = -5Β·sigmoid(a_hΒ·(xΒ·f_a + dt_b)); Ξ² = sigmoid(xΒ·w_Ξ²)
|
| 40028 |
+
* per head; the delta rule on a 128Γ128 state per head, S = diag(e^g)Β·S, S += k β Ξ²(v β Sα΅k), o = Sα΅q / β128
|
| 40029 |
+
* (ggml_gated_delta_net, KDA branch); then RMSNorm(o)Β·w per head times sigmoid(xΒ·g_a), and the output projection.
|
| 40030 |
+
* MLA (multi-head latent attention, 32 heads, absorbed): q = xΒ·W_q (per head 128 no-RoPE + 64 RoPE dims);
|
| 40031 |
+
* [c | k_pe] = xΒ·W_kv_a (512 + 64); c = RMSNorm(c)Β·w; RoPE (GGML NORM: adjacent pairs, base 6e6) on q_pe and k_pe;
|
| 40032 |
+
* the cache keeps [c | k_pe] per token (576 f32). Scores (W_k_b,hΒ·q_nope,h)Β·c + q_peΒ·k_pe, over β192; softmax;
|
| 40033 |
+
* o_h = W_v_b,hΒ·Ξ£ pΒ·c; o_h scaled by sigmoid(xΒ·w_gate)_h; output projection 4096 β 2560.
|
| 40034 |
+
* FFN: layers 0β1 a dense SwiGLU (6144). Layers 2β41: 512 experts in 8 groups of 64; p = sigmoid(router);
|
| 40035 |
+
* selection score p + bias; the 4 groups with the highest sum of their top-2 scores; top 8 experts of those by
|
| 40036 |
+
* score; weights = their p over the sum (β₯ 6.1e-5), Γ 2.5; SwiGLU experts (768) plus one shared expert (768).
|
| 40037 |
+
* Some layers clamp the SwiGLU (llama.cpp's non-fused form): up to [-L, L], silu(gate) to β€ L.
|
| 40038 |
+
* Embedding: Q4_K rows; head: RMSNorm Β· output_norm, then output.weight (Q6_K, untied).
|
| 40039 |
+
*
|
| 40040 |
+
* Decode is one token at a time and prefill feeds tokens through the same step. The recurrent KDA state only moves
|
| 40041 |
+
* forward, so a prompt that does not extend the cached tokens is fed again from position 0.
|
| 40042 |
+
*
|
| 40043 |
+
* Two-part GGUF: the engine reads both part headers first and gives the store one virtual file, part 1's bytes then
|
| 40044 |
+
* part 2's (splitFetch answers range requests across the boundary); tensors of part 2 get offsets in that space.
|
| 40045 |
+
*/
|
| 40046 |
+
|
| 40047 |
+
import { Qwen3MoeSsd, BpeTokenizer, GGML, TOK, STORAGE, COPY_SRC, COPY_DST, MAP_READ, parseGguf, STORE_FORMAT, STORE_ROOT } from './qwen3_moe_ssd.js';
|
| 40048 |
+
import { readStore } from './ingest.js';
|
| 40049 |
+
import { QTYPES, QTYPE_BY_ID, rowBytes, mvKernel, embedKernel, deqFn, gpuBlock, gridData } from './iq_quants.js';
|
| 40050 |
+
|
| 40051 |
+
// inclusionAI's own Q4_K_M GGUF (rev 29edce5, 2026-08-31); sha256 = the LFS object ids of the two parts.
|
| 40052 |
+
export const LING_FLASH = {
|
| 40053 |
+
repo: 'inclusionAI/Ling-3.0-flash-GGUF',
|
| 40054 |
+
revision: '29edce5130491e638f3e5ebec91fa0a5ed4f780e',
|
| 40055 |
+
file: 'Q4_K_M/Ling-3.0-flash-Q4_K_M-00001-of-00002.gguf',
|
| 40056 |
+
parts: [
|
| 40057 |
+
{ file: 'Q4_K_M/Ling-3.0-flash-Q4_K_M-00001-of-00002.gguf', size: 49995534208, sha256: '24ff06f1ab74546a91f7c07bd725b3015896bbe9dd82ab0e5838be9ed06a6dcf' },
|
| 40058 |
+
{ file: 'Q4_K_M/Ling-3.0-flash-Q4_K_M-00002-of-00002.gguf', size: 27014610464, sha256: '1a6a8c91562b817fd8ec5b250453be696b0fc209ed8f2acdb6c13f4e17f7e188' },
|
| 40059 |
+
],
|
| 40060 |
+
size: 49995534208 + 27014610464,
|
| 40061 |
+
};
|
| 40062 |
+
|
| 40063 |
+
// ββ Two-part GGUF as one file ββ
|
| 40064 |
+
// A fetch-shaped function over the parts laid end to end: a Range request is split at the part boundaries and the
|
| 40065 |
+
// bodies are streamed back in order. content-range reports the total size.
|
| 40066 |
+
export function splitFetch(urls, sizes, fetchFn = (u, i) => fetch(u, i)) {
|
| 40067 |
+
const starts = sizes.map((_, i) => sizes.slice(0, i).reduce((a, b) => a + b, 0)), total = sizes.reduce((a, b) => a + b, 0);
|
| 40068 |
+
return async (_url, init = {}) => {
|
| 40069 |
+
const m = /^bytes=(\d+)-(\d+)$/.exec((init.headers && init.headers.Range) || '');
|
| 40070 |
+
if (!m) throw new Error('splitFetch: only Range requests (bytes=a-b) are supported');
|
| 40071 |
+
const a = Number(m[1]), b = Math.min(Number(m[2]), total - 1);
|
| 40072 |
+
const pieces = [];
|
| 40073 |
+
for (let i = 0; i < urls.length; i++) {
|
| 40074 |
+
const lo = Math.max(a, starts[i]), hi = Math.min(b, starts[i] + sizes[i] - 1);
|
| 40075 |
+
if (lo <= hi) pieces.push([i, lo - starts[i], hi - starts[i]]);
|
| 40076 |
+
}
|
| 40077 |
+
const responses = [];
|
| 40078 |
+
for (const [i, lo, hi] of pieces) {
|
| 40079 |
+
const r = await fetchFn(urls[i], { ...init, headers: { ...init.headers, Range: `bytes=${lo}-${hi}` } });
|
| 40080 |
+
if (r.status !== 206) throw new Error(`part ${i + 1} range ${lo}-${hi}: HTTP ${r.status}`);
|
| 40081 |
+
responses.push(r);
|
| 40082 |
+
}
|
| 40083 |
+
if (responses.length === 1) {
|
| 40084 |
+
return new Response(responses[0].body, { status: 206, headers: { 'content-range': `bytes ${a}-${b}/${total}`, ...(responses[0].headers.get('etag') ? { etag: responses[0].headers.get('etag') } : {}) } });
|
| 40085 |
+
}
|
| 40086 |
+
let k = 0, reader = null;
|
| 40087 |
+
const body = new ReadableStream({
|
| 40088 |
+
async pull(ctl) {
|
| 40089 |
+
for (;;) {
|
| 40090 |
+
if (!reader) { if (k >= responses.length) { ctl.close(); return; } reader = responses[k++].body.getReader(); }
|
| 40091 |
+
const { done, value } = await reader.read();
|
| 40092 |
+
if (done) { reader = null; continue; }
|
| 40093 |
+
ctl.enqueue(value); return;
|
| 40094 |
+
}
|
| 40095 |
+
},
|
| 40096 |
+
cancel() { for (const r of responses) try { r.body.cancel(); } catch (_) {} },
|
| 40097 |
+
});
|
| 40098 |
+
return new Response(body, { status: 206, headers: { 'content-range': `bytes ${a}-${b}/${total}` } });
|
| 40099 |
+
};
|
| 40100 |
+
}
|
| 40101 |
+
|
| 40102 |
+
// Reads part 2's header (the parts after the first carry only tensor infos) β its tensors with offsets made relative
|
| 40103 |
+
// to part 1's data start in the virtual file.
|
| 40104 |
+
export async function readPartTensors(url, partStart, dataStart1, fetchFn) {
|
| 40105 |
+
let n = 4 << 20;
|
| 40106 |
+
for (;;) {
|
| 40107 |
+
const r = await fetchFn(url, { headers: { Range: `bytes=0-${n - 1}` }, cache: 'no-store' });
|
| 40108 |
+
if (!r.ok) throw new Error(`GGUF part header fetch: HTTP ${r.status}`);
|
| 40109 |
+
const u8 = new Uint8Array(await r.arrayBuffer());
|
| 40110 |
+
try {
|
| 40111 |
+
const g = parseGguf(u8);
|
| 40112 |
+
return g.tensors.map((t) => ({ ...t, offset: partStart + g.dataStart + t.offset - dataStart1 }));
|
| 40113 |
+
} catch (e) { if (!e.needBytes) throw e; n = e.needBytes; }
|
| 40114 |
+
}
|
| 40115 |
+
}
|
| 40116 |
+
|
| 40117 |
+
// ββ Config ββ
|
| 40118 |
+
export function configFromGgufLing(kv) {
|
| 40119 |
+
const a = kv['general.architecture'];
|
| 40120 |
+
if (a !== 'bailingmoe3') throw new Error(`expected a bailingmoe3 GGUF, got ${a}`);
|
| 40121 |
+
const g = (k) => kv[`${a}.${k}`];
|
| 40122 |
+
if (g('expert_gating_func') !== 2) throw new Error(`bailingmoe3 expert_gating_func ${g('expert_gating_func')}: this engine implements sigmoid (2)`);
|
| 40123 |
+
if (!g('expert_weights_norm')) throw new Error('bailingmoe3 without expert_weights_norm: not implemented');
|
| 40124 |
+
if (g('attention.q_lora_rank')) throw new Error('bailingmoe3 with a q LoRA (Ling 3.1): not implemented');
|
| 40125 |
+
if (g('rope.dimension_sections')) throw new Error('bailingmoe3 with mrope sections (the VL file): not implemented');
|
| 40126 |
+
const allLayers = g('block_count'), nextn = g('nextn_predict_layers') || 0, modelLayers = allLayers - nextn;
|
| 40127 |
+
const lead = g('leading_dense_block_count'), layers = modelLayers - lead;
|
| 40128 |
+
const kvH = g('attention.head_count_kv');
|
| 40129 |
+
return {
|
| 40130 |
+
arch: a, modelLayers, lead, layers,
|
| 40131 |
+
mla: Array.from({ length: modelLayers }, (_, l) => (Array.isArray(kvH) ? kvH[l] : kvH) > 0),
|
| 40132 |
+
hidden: g('embedding_length'), denseFf: g('feed_forward_length'), heads: g('attention.head_count'),
|
| 40133 |
+
kdaHd: g('kda.head_dim'), conv: g('ssm.conv_kernel'), gateLow: g('kda.gate_lower_bound'),
|
| 40134 |
+
kvRank: g('attention.kv_lora_rank'), qkHd: g('attention.key_length_mla'), vHd: g('attention.value_length_mla'),
|
| 40135 |
+
nRot: g('rope.dimension_count'), ropeTheta: g('rope.freq_base'),
|
| 40136 |
+
experts: g('expert_count'), topK: g('expert_used_count'), groups: g('expert_group_count'), groupsUsed: g('expert_group_used_count'),
|
| 40137 |
+
expertFf: g('expert_feed_forward_length'), shexpFf: g('expert_shared_feed_forward_length'), expScale: g('expert_weights_scale') ?? 1,
|
| 40138 |
+
clampExp: g('swiglu_clamp_exp') || [], clampShexp: g('swiglu_clamp_shexp') || [],
|
| 40139 |
+
eps: g('attention.layer_norm_rms_epsilon'), contextLength: g('context_length'),
|
| 40140 |
+
vocab: kv['tokenizer.ggml.tokens'].length, bos: kv['tokenizer.ggml.bos_token_id'], eos: kv['tokenizer.ggml.eos_token_id'],
|
| 40141 |
+
// Base-class fields the pool and readback size from.
|
| 40142 |
+
kvHeads: 1, headDim: g('attention.key_length_mla'),
|
| 40143 |
+
};
|
| 40144 |
+
}
|
| 40145 |
+
|
| 40146 |
+
// ββ Store layout ββ
|
| 40147 |
+
// One expert record = 256-byte guS and dS (unused), down rows (dQ), then gate rows and up rows (Q4_K, guQ). Down is
|
| 40148 |
+
// Q6_K on 17 layers and Q4_K on 23, so dQ is sized for Q6_K and a Q4_K layer leaves a gap after its down rows; guQ
|
| 40149 |
+
// last means every record ends in written bytes, so the file reaches its full length. Dense tensors are raw copies;
|
| 40150 |
+
// the MTP layer is skipped.
|
| 40151 |
+
const KEEP_TYPES = new Set([GGML.F32, QTYPES.Q4_K.id, QTYPES.Q6_K.id, QTYPES.Q5_0.id]);
|
| 40152 |
+
const typeName = (id) => (id === GGML.F32 ? 'f32' : QTYPE_BY_ID[id]);
|
| 40153 |
+
const nBytes = (t) => { const n = t.dims.reduce((x, y) => x * y, 1); return t.type === GGML.F32 ? n * 4 : (n / QTYPES[QTYPE_BY_ID[t.type]].qk) * QTYPES[QTYPE_BY_ID[t.type]].block; };
|
| 40154 |
+
|
| 40155 |
+
export function planLayoutLing(gguf) {
|
| 40156 |
+
const cfg = configFromGgufLing(gguf.kv), H = cfg.hidden, F = cfg.expertFf;
|
| 40157 |
+
const gu = 2 * F * rowBytes('Q4_K', H), dn = Math.max(H * rowBytes('Q6_K', F), H * rowBytes('Q4_K', F));
|
| 40158 |
+
const parts = { guS: { off: 0, bytes: 256 }, dS: { off: 256, bytes: 256 }, dQ: { off: 512, bytes: dn }, guQ: { off: 512 + dn, bytes: gu } };
|
| 40159 |
+
const record = parts.guQ.off + parts.guQ.bytes;
|
| 40160 |
+
const dense = {}, downType = [];
|
| 40161 |
+
let off = 0;
|
| 40162 |
+
const take = (n) => { const o = off; off += Math.ceil(n / 256) * 256; return o; };
|
| 40163 |
+
for (const t of gguf.tensors) {
|
| 40164 |
+
const layer = /^blk\.(\d+)\./.exec(t.name);
|
| 40165 |
+
if (layer && Number(layer[1]) >= cfg.modelLayers) continue; // the MTP head
|
| 40166 |
+
if (!KEEP_TYPES.has(t.type)) throw new Error(`${t.name}: unsupported type ${t.type} (this engine reads inclusionAI's Q4_K_M file)`);
|
| 40167 |
+
const m = /^blk\.(\d+)\.ffn_(gate|up|down)_exps\.weight$/.exec(t.name);
|
| 40168 |
+
if (m) {
|
| 40169 |
+
const want = m[2] === 'down' ? null : QTYPES.Q4_K.id;
|
| 40170 |
+
if (want !== null && t.type !== want) throw new Error(`${t.name}: experts gate/up must be Q4_K`);
|
| 40171 |
+
if (m[2] === 'down') downType[Number(m[1]) - cfg.lead] = typeName(t.type);
|
| 40172 |
+
continue;
|
| 40173 |
+
}
|
| 40174 |
+
const n = nBytes(t);
|
| 40175 |
+
dense[t.name] = { type: typeName(t.type), dims: t.dims, raw: { off: take(n), bytes: n } };
|
| 40176 |
+
}
|
| 40177 |
+
for (let l = 0; l < cfg.layers; l++) if (!downType[l]) throw new Error(`bailingmoe3: no ffn_down_exps for layer ${l + cfg.lead}`);
|
| 40178 |
+
return {
|
| 40179 |
+
config: cfg, downType,
|
| 40180 |
+
experts: { file: 'experts.bin', record, parts, layers: cfg.layers, perLayer: cfg.experts, bytes: record * cfg.layers * cfg.experts },
|
| 40181 |
+
dense: { file: 'dense.bin', bytes: off, tensors: dense },
|
| 40182 |
+
};
|
| 40183 |
+
}
|
| 40184 |
+
|
| 40185 |
+
// Every unit is a raw copy: expert slices to their record, dense tensors in bands of at most 8 MB.
|
| 40186 |
+
export function planUnitsLing(gguf, layout) {
|
| 40187 |
+
const units = [];
|
| 40188 |
+
const { record, parts, perLayer } = layout.experts, { lead, modelLayers, hidden: H, expertFf: F } = layout.config;
|
| 40189 |
+
const gateBytes = F * rowBytes('Q4_K', H), band = 8 << 20;
|
| 40190 |
+
for (const t of gguf.tensors) {
|
| 40191 |
+
const abs = gguf.dataStart + t.offset;
|
| 40192 |
+
const layer = /^blk\.(\d+)\./.exec(t.name);
|
| 40193 |
+
if (layer && Number(layer[1]) >= modelLayers) continue;
|
| 40194 |
+
const m = /^blk\.(\d+)\.ffn_(gate|up|down)_exps\.weight$/.exec(t.name);
|
| 40195 |
+
if (m) {
|
| 40196 |
+
const l = Number(m[1]) - lead, slice = nBytes(t) / perLayer;
|
| 40197 |
+
const dst = m[2] === 'down' ? parts.dQ.off : parts.guQ.off + (m[2] === 'up' ? gateBytes : 0);
|
| 40198 |
+
for (let e = 0; e < perLayer; e++) units.push({ src: abs + e * slice, len: slice, file: 'experts', raw: (l * perLayer + e) * record + dst });
|
| 40199 |
+
continue;
|
| 40200 |
+
}
|
| 40201 |
+
const d = layout.dense.tensors[t.name];
|
| 40202 |
+
for (let o = 0; o < d.raw.bytes; o += band) units.push({ src: abs + o, len: Math.min(band, d.raw.bytes - o), file: 'dense', raw: d.raw.off + o });
|
| 40203 |
+
}
|
| 40204 |
+
units.sort((a, b) => a.src - b.src);
|
| 40205 |
+
return units;
|
| 40206 |
+
}
|
| 40207 |
+
|
| 40208 |
+
// ββ Tokenizer: byte-level BPE, llama.cpp LLAMA_VOCAB_PRE_TYPE_BAILINGMOE (tokenizer.ggml.pre = bailingmoe2): the Qwen2
|
| 40209 |
+
// split with its newline piece written \s*[\r\n] (Qwen2: \s*[\r\n]+). Possessive quantifiers spelled greedy, as llama.cpp.
|
| 40210 |
+
const BAILING_PRE = /(?:'[sSdDmMtT]|'[lL][lL]|'[vV][eE]|'[rR][eE])|[^\r\n\p{L}\p{N}]?\p{L}+|\p{N}| ?[^\s\p{L}\p{N}]+[\r\n]*|\s*[\r\n]|\s+(?!\S)|\s+/gu;
|
| 40211 |
+
export class LingTokenizer extends BpeTokenizer {
|
| 40212 |
+
constructor(kv) { super(kv); this.preRe = BAILING_PRE; }
|
| 40213 |
+
}
|
| 40214 |
+
|
| 40215 |
+
// The GGUF's Bailing V3 template (no tools): '<role>SYSTEM</role>' + system text + 'detailed thinking on|off' +
|
| 40216 |
+
// '<|role_end|>', then '<role>HUMAN</role>β¦<|role_end|>' and '<role>ASSISTANT</role>\n<think>β¦</think>β¦<|role_end|>',
|
| 40217 |
+
// ending in '<role>ASSISTANT</role>\n<think>' (thinking on) or 'β¦\n<think></think>' (off). Earlier replies keep their
|
| 40218 |
+
// reasoning (preserved_thinking).
|
| 40219 |
+
export function lingChatPrompt(messages, { enableThinking = true } = {}) {
|
| 40220 |
+
const opt = enableThinking ? 'on' : 'off';
|
| 40221 |
+
const first = messages[0] && messages[0].role === 'system' ? messages[0] : null;
|
| 40222 |
+
const text = (c) => (typeof c === 'string' ? c : '');
|
| 40223 |
+
let s = '<role>SYSTEM</role>';
|
| 40224 |
+
if (first && /detailed thinking (on|off)/.test(text(first.content))) s += text(first.content) + '<|role_end|>';
|
| 40225 |
+
else s += (first ? text(first.content) + '\n' : '') + `detailed thinking ${opt}<|role_end|>`;
|
| 40226 |
+
for (let i = first ? 1 : 0; i < messages.length; i++) {
|
| 40227 |
+
const m = messages[i], c = text(m.content);
|
| 40228 |
+
if (m.role === 'user') s += '<role>HUMAN</role>' + c + '<|role_end|>';
|
| 40229 |
+
else if (m.role === 'system') s += '<role>SYSTEM</role>' + c + '<|role_end|>';
|
| 40230 |
+
else if (m.role === 'assistant') {
|
| 40231 |
+
let reasoning = typeof m.reasoning_content === 'string' ? m.reasoning_content : '', content = c;
|
| 40232 |
+
if (!reasoning && content.includes('</think>')) {
|
| 40233 |
+
reasoning = content.split('</think>')[0].replace(/\n+$/, '').split('<think>').pop().replace(/^\n+/, '');
|
| 40234 |
+
content = content.split('</think>').pop().replace(/^\n+/, '');
|
| 40235 |
+
}
|
| 40236 |
+
s += reasoning ? '<role>ASSISTANT</role>\n<think>' + reasoning.replace(/^\n+|\n+$/g, '') + '</think>' + content.replace(/^\n+/, '')
|
| 40237 |
+
: '<role>ASSISTANT</role>\n<think></think>' + content;
|
| 40238 |
+
s += '<|role_end|>';
|
| 40239 |
+
} else if (m.role === 'tool') {
|
| 40240 |
+
if (i === 0 || messages[i - 1].role !== 'tool') s += '<role>OBSERVATION</role>';
|
| 40241 |
+
s += '\n<tool_response>\n' + c + '\n</tool_response>';
|
| 40242 |
+
if (i === messages.length - 1 || messages[i + 1].role !== 'tool') s += '<|role_end|>';
|
| 40243 |
+
}
|
| 40244 |
+
}
|
| 40245 |
+
return s + '<role>ASSISTANT</role>' + (enableThinking ? '\n<think>' : '\n<think></think>');
|
| 40246 |
+
}
|
| 40247 |
+
|
| 40248 |
+
// ββ Kernels ββ
|
| 40249 |
+
const SIGM = 'fn sigm(x: f32) -> f32 { return 1.0 / (1.0 + exp(-x)); }';
|
| 40250 |
+
const LK = {
|
| 40251 |
+
// Debug capture (opts.captureX): dst[offΒ·n + i] = x[i].
|
| 40252 |
+
copyAt: `
|
| 40253 |
+
struct P { n: u32, off: u32 }
|
| 40254 |
+
@group(0) @binding(0) var<storage, read> x: array<f32>;
|
| 40255 |
+
@group(0) @binding(1) var<storage, read_write> dst: array<f32>;
|
| 40256 |
+
@group(0) @binding(2) var<uniform> p: P;
|
| 40257 |
+
@compute @workgroup_size(256) fn main(@builtin(global_invocation_id) g: vec3<u32>) { if (g.x < p.n) { dst[p.off * p.n + g.x] = x[g.x]; } }`,
|
| 40258 |
+
// x += a (n values); also used for the shared expert's output.
|
| 40259 |
+
addInPlace: `
|
| 40260 |
+
struct P { n: u32 }
|
| 40261 |
+
@group(0) @binding(0) var<storage, read_write> x: array<f32>;
|
| 40262 |
+
@group(0) @binding(1) var<storage, read> a: array<f32>;
|
| 40263 |
+
@group(0) @binding(2) var<uniform> p: P;
|
| 40264 |
+
@compute @workgroup_size(256) fn main(@builtin(global_invocation_id) g: vec3<u32>) { if (g.x < p.n) { x[g.x] = x[g.x] + a[g.x]; } }`,
|
| 40265 |
+
// act[zΒ·F + j] = min(silu(g), L)Β·clamp(u, -L, L) with g = gu[zΒ·2F + j], u = gu[zΒ·2F + F + j]; L = 0 means no clamp.
|
| 40266 |
+
swiglu: `
|
| 40267 |
+
struct P { F: u32, k: u32, L: f32, pad: u32 }
|
| 40268 |
+
@group(0) @binding(0) var<storage, read> gu: array<f32>;
|
| 40269 |
+
@group(0) @binding(1) var<storage, read_write> act: array<f32>;
|
| 40270 |
+
@group(0) @binding(2) var<uniform> p: P;
|
| 40271 |
+
@compute @workgroup_size(256) fn main(@builtin(global_invocation_id) g: vec3<u32>) {
|
| 40272 |
+
let i = g.x; if (i >= p.k * p.F) { return; }
|
| 40273 |
+
let z = i / p.F; let j = i % p.F;
|
| 40274 |
+
let g0 = gu[z * 2u * p.F + j]; var u = gu[z * 2u * p.F + p.F + j];
|
| 40275 |
+
var sg = g0 / (1.0 + exp(-g0));
|
| 40276 |
+
if (p.L > 0.0) { u = clamp(u, -p.L, p.L); sg = min(sg, p.L); }
|
| 40277 |
+
act[i] = sg * u;
|
| 40278 |
+
}`,
|
| 40279 |
+
// x[h] += Ξ£_j w_j Β· out[jΒ·H + h] + shared[h]; weights are the selection's second half.
|
| 40280 |
+
moeAccum: `
|
| 40281 |
+
struct P { H: u32, k: u32 }
|
| 40282 |
+
@group(0) @binding(0) var<storage, read> out: array<f32>;
|
| 40283 |
+
@group(0) @binding(1) var<storage, read> sel: array<u32>;
|
| 40284 |
+
@group(0) @binding(2) var<storage, read> sh: array<f32>;
|
| 40285 |
+
@group(0) @binding(3) var<storage, read_write> x: array<f32>;
|
| 40286 |
+
@group(0) @binding(4) var<uniform> p: P;
|
| 40287 |
+
@compute @workgroup_size(256) fn main(@builtin(global_invocation_id) g: vec3<u32>) {
|
| 40288 |
+
let h = g.x; if (h >= p.H) { return; }
|
| 40289 |
+
var acc = 0.0;
|
| 40290 |
+
for (var j = 0u; j < p.k; j++) { acc += bitcast<f32>(sel[p.k + j]) * out[j * p.H + h]; }
|
| 40291 |
+
x[h] = x[h] + (acc + sh[h]);
|
| 40292 |
+
}`,
|
| 40293 |
+
// Group-limited sigmoid routing (build_moe_ffn with exp_probs_b and n_expert_groups). 512 experts, 2 per thread.
|
| 40294 |
+
// sel[0..k) = expert ids, sel[k..2k) = weights (f32 bits).
|
| 40295 |
+
routeLing: `
|
| 40296 |
+
struct P { n: u32, k: u32, groups: u32, used: u32, scale: f32, pad0: u32, pad1: u32, pad2: u32 }
|
| 40297 |
+
@group(0) @binding(0) var<storage, read> lg: array<f32>;
|
| 40298 |
+
@group(0) @binding(1) var<storage, read> bias: array<f32>;
|
| 40299 |
+
@group(0) @binding(2) var<storage, read_write> sel: array<u32>;
|
| 40300 |
+
@group(0) @binding(3) var<uniform> p: P;
|
| 40301 |
+
${SIGM}
|
| 40302 |
+
var<workgroup> sc: array<f32, 512>;
|
| 40303 |
+
var<workgroup> gs: array<f32, 16>;
|
| 40304 |
+
var<workgroup> keep: array<u32, 16>;
|
| 40305 |
+
var<workgroup> red: array<f32, 256>;
|
| 40306 |
+
var<workgroup> bi: array<u32, 256>;
|
| 40307 |
+
var<workgroup> picks: array<u32, 32>;
|
| 40308 |
+
@compute @workgroup_size(256) fn main(@builtin(local_invocation_id) l: vec3<u32>) {
|
| 40309 |
+
let t = l.x; let per = p.n / p.groups;
|
| 40310 |
+
for (var e = t; e < p.n; e += 256u) { sc[e] = sigm(lg[e]) + bias[e]; }
|
| 40311 |
+
workgroupBarrier();
|
| 40312 |
+
if (t < p.groups) {
|
| 40313 |
+
var a = -3.4e38; var b = -3.4e38;
|
| 40314 |
+
for (var e = t * per; e < (t + 1u) * per; e++) { let v = sc[e]; if (v > a) { b = a; a = v; } else if (v > b) { b = v; } }
|
| 40315 |
+
gs[t] = a + b; keep[t] = 0u;
|
| 40316 |
+
}
|
| 40317 |
+
workgroupBarrier();
|
| 40318 |
+
if (t == 0u) {
|
| 40319 |
+
for (var j = 0u; j < p.used; j++) {
|
| 40320 |
+
var best = 0u; var bv = -3.4e38;
|
| 40321 |
+
for (var q = 0u; q < p.groups; q++) { if (keep[q] == 0u && gs[q] > bv) { bv = gs[q]; best = q; } }
|
| 40322 |
+
keep[best] = 1u;
|
| 40323 |
+
}
|
| 40324 |
+
}
|
| 40325 |
+
workgroupBarrier();
|
| 40326 |
+
var m0 = select(-3.4e38, sc[t], keep[t / per] == 1u);
|
| 40327 |
+
var m1 = select(-3.4e38, sc[t + 256u], keep[(t + 256u) / per] == 1u);
|
| 40328 |
+
for (var j = 0u; j < p.k; j++) {
|
| 40329 |
+
if (m0 >= m1) { red[t] = m0; bi[t] = t; } else { red[t] = m1; bi[t] = t + 256u; }
|
| 40330 |
+
workgroupBarrier();
|
| 40331 |
+
for (var s = 128u; s > 0u; s >>= 1u) {
|
| 40332 |
+
if (t < s) { let o = red[t + s]; if (o > red[t] || (o == red[t] && bi[t + s] < bi[t])) { red[t] = o; bi[t] = bi[t + s]; } }
|
| 40333 |
+
workgroupBarrier();
|
| 40334 |
+
}
|
| 40335 |
+
if (t == 0u) { picks[j] = bi[0]; }
|
| 40336 |
+
workgroupBarrier();
|
| 40337 |
+
if (picks[j] == t) { m0 = -3.4e38; }
|
| 40338 |
+
if (picks[j] == t + 256u) { m1 = -3.4e38; }
|
| 40339 |
+
workgroupBarrier();
|
| 40340 |
+
}
|
| 40341 |
+
if (t == 0u) {
|
| 40342 |
+
var wsum = 0.0;
|
| 40343 |
+
for (var j = 0u; j < p.k; j++) { wsum += sigm(lg[picks[j]]); }
|
| 40344 |
+
wsum = max(wsum, 6.103515625e-5);
|
| 40345 |
+
for (var j = 0u; j < p.k; j++) { sel[j] = picks[j]; sel[p.k + j] = bitcast<u32>((sigm(lg[picks[j]]) / wsum) * p.scale); }
|
| 40346 |
+
}
|
| 40347 |
+
}`,
|
| 40348 |
+
// KDA short conv, one token: for channel c, y = silu(Ξ£_{i<3} w[4c+i]Β·st[3c+i] + w[4c+3]Β·x[c]); st shifts by one.
|
| 40349 |
+
convStep: `
|
| 40350 |
+
struct P { n: u32 }
|
| 40351 |
+
@group(0) @binding(0) var<storage, read> x: array<f32>;
|
| 40352 |
+
@group(0) @binding(1) var<storage, read> w: array<f32>;
|
| 40353 |
+
@group(0) @binding(2) var<storage, read_write> st: array<f32>;
|
| 40354 |
+
@group(0) @binding(3) var<storage, read_write> y: array<f32>;
|
| 40355 |
+
@group(0) @binding(4) var<uniform> p: P;
|
| 40356 |
+
@compute @workgroup_size(256) fn main(@builtin(global_invocation_id) g: vec3<u32>) {
|
| 40357 |
+
let c = g.x; if (c >= p.n) { return; }
|
| 40358 |
+
let s0 = st[3u * c]; let s1 = st[3u * c + 1u]; let s2 = st[3u * c + 2u]; let xc = x[c];
|
| 40359 |
+
let a = w[4u * c] * s0 + w[4u * c + 1u] * s1 + w[4u * c + 2u] * s2 + w[4u * c + 3u] * xc;
|
| 40360 |
+
st[3u * c] = s1; st[3u * c + 1u] = s2; st[3u * c + 2u] = xc;
|
| 40361 |
+
y[c] = a / (1.0 + exp(-a));
|
| 40362 |
+
}`,
|
| 40363 |
+
// KDA gates: g[c] = loΒ·sigmoid(a[c / hd]Β·(fa[c] + dtb[c])) (c < HΒ·hd), beta[h] = sigmoid(bl[h]) (stored after g).
|
| 40364 |
+
kdaGates: `
|
| 40365 |
+
struct P { n: u32, hd: u32, heads: u32, lo: f32 }
|
| 40366 |
+
@group(0) @binding(0) var<storage, read> fa: array<f32>;
|
| 40367 |
+
@group(0) @binding(1) var<storage, read> dtb: array<f32>;
|
| 40368 |
+
@group(0) @binding(2) var<storage, read> A: array<f32>;
|
| 40369 |
+
@group(0) @binding(3) var<storage, read> bl: array<f32>;
|
| 40370 |
+
@group(0) @binding(4) var<storage, read_write> gb: array<f32>;
|
| 40371 |
+
@group(0) @binding(5) var<uniform> p: P;
|
| 40372 |
+
${SIGM}
|
| 40373 |
+
@compute @workgroup_size(256) fn main(@builtin(global_invocation_id) g: vec3<u32>) {
|
| 40374 |
+
let c = g.x;
|
| 40375 |
+
if (c < p.n) { gb[c] = p.lo * sigm(A[c / p.hd] * (fa[c] + dtb[c])); }
|
| 40376 |
+
if (c < p.heads) { gb[p.n + c] = sigm(bl[c]); }
|
| 40377 |
+
}`,
|
| 40378 |
+
// KDA delta rule, one token; one workgroup of 128 per head, lane j owns column j of S (st[h][j][i] = S[i][j]).
|
| 40379 |
+
// q and k are L2-normalised here (x / β(Ξ£xΒ² + eps)); gb holds g (HΒ·hd) then Ξ² (H).
|
| 40380 |
+
kdaStep: `
|
| 40381 |
+
struct P { hd: u32, n: u32, eps: f32, scale: f32 }
|
| 40382 |
+
@group(0) @binding(0) var<storage, read> q: array<f32>;
|
| 40383 |
+
@group(0) @binding(1) var<storage, read> k: array<f32>;
|
| 40384 |
+
@group(0) @binding(2) var<storage, read> v: array<f32>;
|
| 40385 |
+
@group(0) @binding(3) var<storage, read> gb: array<f32>;
|
| 40386 |
+
@group(0) @binding(4) var<storage, read_write> st: array<f32>;
|
| 40387 |
+
@group(0) @binding(5) var<storage, read_write> o: array<f32>;
|
| 40388 |
+
@group(0) @binding(6) var<uniform> p: P;
|
| 40389 |
+
var<workgroup> qs: array<f32, 128>;
|
| 40390 |
+
var<workgroup> ks: array<f32, 128>;
|
| 40391 |
+
var<workgroup> eg: array<f32, 128>;
|
| 40392 |
+
var<workgroup> red: array<f32, 128>;
|
| 40393 |
+
var<workgroup> red2: array<f32, 128>;
|
| 40394 |
+
@compute @workgroup_size(128) fn main(@builtin(workgroup_id) wg: vec3<u32>, @builtin(local_invocation_id) l: vec3<u32>) {
|
| 40395 |
+
let h = wg.x; let j = l.x; let b = h * 128u;
|
| 40396 |
+
let qv = q[b + j]; let kv = k[b + j];
|
| 40397 |
+
red[j] = qv * qv; red2[j] = kv * kv; workgroupBarrier();
|
| 40398 |
+
for (var s = 64u; s > 0u; s >>= 1u) { if (j < s) { red[j] += red[j + s]; red2[j] += red2[j + s]; } workgroupBarrier(); }
|
| 40399 |
+
qs[j] = qv / sqrt(red[0] + p.eps); ks[j] = kv / sqrt(red2[0] + p.eps); eg[j] = exp(gb[b + j]);
|
| 40400 |
+
workgroupBarrier();
|
| 40401 |
+
let beta = gb[p.n + h];
|
| 40402 |
+
let row = (h * 128u + j) * 128u;
|
| 40403 |
+
var sk = 0.0;
|
| 40404 |
+
for (var i = 0u; i < 128u; i++) { let m = st[row + i] * eg[i]; st[row + i] = m; sk += m * ks[i]; }
|
| 40405 |
+
let d = (v[b + j] - sk) * beta;
|
| 40406 |
+
var acc = 0.0;
|
| 40407 |
+
for (var i = 0u; i < 128u; i++) { let m = st[row + i] + ks[i] * d; st[row + i] = m; acc += m * qs[i]; }
|
| 40408 |
+
o[b + j] = acc * p.scale;
|
| 40409 |
+
}`,
|
| 40410 |
+
// y = RMSNorm(o_h)Β·w Β· sigmoid(gz) per head of hd (one workgroup of 128 per head).
|
| 40411 |
+
headNormGate: `
|
| 40412 |
+
struct P { hd: u32, eps: f32 }
|
| 40413 |
+
@group(0) @binding(0) var<storage, read> o: array<f32>;
|
| 40414 |
+
@group(0) @binding(1) var<storage, read> w: array<f32>;
|
| 40415 |
+
@group(0) @binding(2) var<storage, read> gz: array<f32>;
|
| 40416 |
+
@group(0) @binding(3) var<storage, read_write> y: array<f32>;
|
| 40417 |
+
@group(0) @binding(4) var<uniform> p: P;
|
| 40418 |
+
${SIGM}
|
| 40419 |
+
var<workgroup> red: array<f32, 128>;
|
| 40420 |
+
@compute @workgroup_size(128) fn main(@builtin(workgroup_id) wg: vec3<u32>, @builtin(local_invocation_id) l: vec3<u32>) {
|
| 40421 |
+
let i = wg.x * p.hd + l.x; let v = o[i];
|
| 40422 |
+
red[l.x] = v * v; workgroupBarrier();
|
| 40423 |
+
for (var s = 64u; s > 0u; s >>= 1u) { if (l.x < s) { red[l.x] += red[l.x + s]; } workgroupBarrier(); }
|
| 40424 |
+
y[i] = (v / sqrt(red[0] / f32(p.hd) + p.eps)) * w[l.x] * sigm(gz[i]);
|
| 40425 |
+
}`,
|
| 40426 |
+
// MLA: the cache row for this token: c = RMSNorm(kva[0..R))Β·w, k_pe = RoPE(kva[R..R+nRot)); one workgroup.
|
| 40427 |
+
mlaKv: `
|
| 40428 |
+
${TOK}
|
| 40429 |
+
struct P { R: u32, nRot: u32, eps: f32, pad: u32 }
|
| 40430 |
+
@group(0) @binding(0) var<storage, read> kva: array<f32>;
|
| 40431 |
+
@group(0) @binding(1) var<storage, read> w: array<f32>;
|
| 40432 |
+
@group(0) @binding(2) var<storage, read> rope: array<f32>;
|
| 40433 |
+
@group(0) @binding(3) var<storage, read_write> cache: array<f32>;
|
| 40434 |
+
@group(0) @binding(4) var<uniform> tok: Tok;
|
| 40435 |
+
@group(0) @binding(5) var<uniform> p: P;
|
| 40436 |
+
var<workgroup> red: array<f32, 256>;
|
| 40437 |
+
@compute @workgroup_size(256) fn main(@builtin(local_invocation_id) l: vec3<u32>) {
|
| 40438 |
+
let t = l.x; var acc = 0.0;
|
| 40439 |
+
for (var i = t; i < p.R; i += 256u) { let v = kva[i]; acc += v * v; }
|
| 40440 |
+
red[t] = acc; workgroupBarrier();
|
| 40441 |
+
for (var s = 128u; s > 0u; s >>= 1u) { if (t < s) { red[t] += red[t + s]; } workgroupBarrier(); }
|
| 40442 |
+
let scale = 1.0 / sqrt(red[0] / f32(p.R) + p.eps);
|
| 40443 |
+
let base = tok.pos * (p.R + p.nRot);
|
| 40444 |
+
for (var i = t; i < p.R; i += 256u) { cache[base + i] = (kva[i] * scale) * w[i]; }
|
| 40445 |
+
if (t < p.nRot / 2u) {
|
| 40446 |
+
let a = kva[p.R + 2u * t]; let b = kva[p.R + 2u * t + 1u]; let c = rope[t]; let sn = rope[p.nRot / 2u + t];
|
| 40447 |
+
cache[base + p.R + 2u * t] = a * c - b * sn;
|
| 40448 |
+
cache[base + p.R + 2u * t + 1u] = a * sn + b * c;
|
| 40449 |
+
}
|
| 40450 |
+
}`,
|
| 40451 |
+
// MLA query: qa[h][r] = Ξ£_d wkb[h][r][d]Β·q[hΒ·qk + d] (d < nope), and the RoPE'd q_pe[h] after it: qa row = R + nRot.
|
| 40452 |
+
// One thread per (h, r) for r < R; threads r < nRot/2 of row R.. rotate the q_pe pairs.
|
| 40453 |
+
mlaQ: `
|
| 40454 |
+
struct P { heads: u32, R: u32, nope: u32, nRot: u32 }
|
| 40455 |
+
@group(0) @binding(0) var<storage, read> q: array<f32>;
|
| 40456 |
+
@group(0) @binding(1) var<storage, read> wkb: array<f32>;
|
| 40457 |
+
@group(0) @binding(2) var<storage, read> rope: array<f32>;
|
| 40458 |
+
@group(0) @binding(3) var<storage, read_write> qa: array<f32>;
|
| 40459 |
+
@group(0) @binding(4) var<uniform> p: P;
|
| 40460 |
+
@compute @workgroup_size(256) fn main(@builtin(workgroup_id) wg: vec3<u32>, @builtin(local_invocation_id) l: vec3<u32>) {
|
| 40461 |
+
let h = wg.y; let r = wg.x * 256u + l.x; let qk = p.nope + p.nRot; let qb = h * qk; let ob = h * (p.R + p.nRot);
|
| 40462 |
+
if (r < p.R) {
|
| 40463 |
+
var acc = 0.0; let wb = (h * p.R + r) * p.nope;
|
| 40464 |
+
for (var d = 0u; d < p.nope; d++) { acc += wkb[wb + d] * q[qb + d]; }
|
| 40465 |
+
qa[ob + r] = acc;
|
| 40466 |
+
} else if (r < p.R + p.nRot / 2u) {
|
| 40467 |
+
let i = r - p.R;
|
| 40468 |
+
let a = q[qb + p.nope + 2u * i]; let b = q[qb + p.nope + 2u * i + 1u]; let c = rope[i]; let sn = rope[p.nRot / 2u + i];
|
| 40469 |
+
qa[ob + p.R + 2u * i] = a * c - b * sn;
|
| 40470 |
+
qa[ob + p.R + 2u * i + 1u] = a * sn + b * c;
|
| 40471 |
+
}
|
| 40472 |
+
}`,
|
| 40473 |
+
// Scores sc[hΒ·S + t] = qa[h]Β·cache[t] Β· scale over the R + nRot latent dims, t < seqLen. One thread per (h, t).
|
| 40474 |
+
mlaScore: `
|
| 40475 |
+
${TOK}
|
| 40476 |
+
struct P { heads: u32, D: u32, S: u32, scale: f32 }
|
| 40477 |
+
@group(0) @binding(0) var<storage, read> qa: array<f32>;
|
| 40478 |
+
@group(0) @binding(1) var<storage, read> cache: array<f32>;
|
| 40479 |
+
@group(0) @binding(2) var<storage, read_write> sc: array<f32>;
|
| 40480 |
+
@group(0) @binding(3) var<uniform> tok: Tok;
|
| 40481 |
+
@group(0) @binding(4) var<uniform> p: P;
|
| 40482 |
+
@compute @workgroup_size(256) fn main(@builtin(workgroup_id) wg: vec3<u32>, @builtin(num_workgroups) ng: vec3<u32>, @builtin(local_invocation_id) l: vec3<u32>) {
|
| 40483 |
+
let i = (wg.y * ng.x + wg.x) * 256u + l.x; let h = i / tok.seqLen; let t = i % tok.seqLen;
|
| 40484 |
+
if (h >= p.heads) { return; }
|
| 40485 |
+
var acc = 0.0; let qb = h * p.D; let cb = t * p.D;
|
| 40486 |
+
for (var d = 0u; d < p.D; d++) { acc += qa[qb + d] * cache[cb + d]; }
|
| 40487 |
+
sc[h * p.S + t] = acc * p.scale;
|
| 40488 |
+
}`,
|
| 40489 |
+
// Softmax over sc[hΒ·S .. hΒ·S + seqLen) in place; one workgroup per head.
|
| 40490 |
+
softmaxS: `
|
| 40491 |
+
${TOK}
|
| 40492 |
+
struct P { S: u32 }
|
| 40493 |
+
@group(0) @binding(0) var<storage, read_write> sc: array<f32>;
|
| 40494 |
+
@group(0) @binding(1) var<uniform> tok: Tok;
|
| 40495 |
+
@group(0) @binding(2) var<uniform> p: P;
|
| 40496 |
+
var<workgroup> red: array<f32, 256>;
|
| 40497 |
+
@compute @workgroup_size(256) fn main(@builtin(workgroup_id) wg: vec3<u32>, @builtin(local_invocation_id) l: vec3<u32>) {
|
| 40498 |
+
let b = wg.x * p.S; let n = tok.seqLen; let t = l.x;
|
| 40499 |
+
var mx = -3.4e38; for (var i = t; i < n; i += 256u) { mx = max(mx, sc[b + i]); }
|
| 40500 |
+
red[t] = mx; workgroupBarrier();
|
| 40501 |
+
for (var s = 128u; s > 0u; s >>= 1u) { if (t < s) { red[t] = max(red[t], red[t + s]); } workgroupBarrier(); }
|
| 40502 |
+
let m = red[0]; workgroupBarrier();
|
| 40503 |
+
var sum = 0.0; for (var i = t; i < n; i += 256u) { let e = exp(sc[b + i] - m); sc[b + i] = e; sum += e; }
|
| 40504 |
+
red[t] = sum; workgroupBarrier();
|
| 40505 |
+
for (var s = 128u; s > 0u; s >>= 1u) { if (t < s) { red[t] += red[t + s]; } workgroupBarrier(); }
|
| 40506 |
+
let inv = 1.0 / red[0];
|
| 40507 |
+
for (var i = t; i < n; i += 256u) { sc[b + i] = sc[b + i] * inv; }
|
| 40508 |
+
}`,
|
| 40509 |
+
// ol[h][r] = Ξ£_t sc[hΒ·S + t]Β·cache[t][r], r < R. One thread per (h, r).
|
| 40510 |
+
mlaMix: `
|
| 40511 |
+
${TOK}
|
| 40512 |
+
struct P { heads: u32, R: u32, D: u32, S: u32 }
|
| 40513 |
+
@group(0) @binding(0) var<storage, read> sc: array<f32>;
|
| 40514 |
+
@group(0) @binding(1) var<storage, read> cache: array<f32>;
|
| 40515 |
+
@group(0) @binding(2) var<storage, read_write> ol: array<f32>;
|
| 40516 |
+
@group(0) @binding(3) var<uniform> tok: Tok;
|
| 40517 |
+
@group(0) @binding(4) var<uniform> p: P;
|
| 40518 |
+
@compute @workgroup_size(256) fn main(@builtin(workgroup_id) wg: vec3<u32>, @builtin(local_invocation_id) l: vec3<u32>) {
|
| 40519 |
+
let h = wg.y; let r = wg.x * 256u + l.x; if (r >= p.R) { return; }
|
| 40520 |
+
var acc = 0.0;
|
| 40521 |
+
for (var t = 0u; t < tok.seqLen; t++) { acc += sc[h * p.S + t] * cache[t * p.D + r]; }
|
| 40522 |
+
ol[h * p.R + r] = acc;
|
| 40523 |
+
}`,
|
| 40524 |
+
// o[hΒ·vd + d] = sigmoid(gt[h]) Β· Ξ£_r wvb[h][d][r]Β·ol[h][r]. One thread per (h, d).
|
| 40525 |
+
mlaOut: `
|
| 40526 |
+
struct P { heads: u32, R: u32, vd: u32, pad: u32 }
|
| 40527 |
+
@group(0) @binding(0) var<storage, read> ol: array<f32>;
|
| 40528 |
+
@group(0) @binding(1) var<storage, read> wvb: array<f32>;
|
| 40529 |
+
@group(0) @binding(2) var<storage, read> gt: array<f32>;
|
| 40530 |
+
@group(0) @binding(3) var<storage, read_write> o: array<f32>;
|
| 40531 |
+
@group(0) @binding(4) var<uniform> p: P;
|
| 40532 |
+
${SIGM}
|
| 40533 |
+
@compute @workgroup_size(128) fn main(@builtin(workgroup_id) wg: vec3<u32>, @builtin(local_invocation_id) l: vec3<u32>) {
|
| 40534 |
+
let h = wg.x; let d = l.x; if (d >= p.vd) { return; }
|
| 40535 |
+
var acc = 0.0; let wb = (h * p.vd + d) * p.R; let ob = h * p.R;
|
| 40536 |
+
for (var r = 0u; r < p.R; r++) { acc += wvb[wb + r] * ol[ob + r]; }
|
| 40537 |
+
o[h * p.vd + d] = acc * sigm(gt[h]);
|
| 40538 |
+
}`,
|
| 40539 |
+
};
|
| 40540 |
+
|
| 40541 |
+
// Expert matvec over the pool: z = workgroup_id.z picks the expert, slot = slots[z]; rows start at slotΒ·slotBytes;
|
| 40542 |
+
// x is shared (xStride 0, gateβup) or per expert (xStride F, down). y[zΒ·M + m]. Generated per block type.
|
| 40543 |
+
function expertKernel(type) {
|
| 40544 |
+
const CPB = QTYPES[type].qk / 32;
|
| 40545 |
+
return `
|
| 40546 |
+
struct P { M: u32, N: u32, rowBytes: u32, slotBytes: u32, xStride: u32, pad0: u32, pad1: u32, pad2: u32 }
|
| 40547 |
+
@group(0) @binding(0) var<storage, read> xa: array<f32>;
|
| 40548 |
+
@group(0) @binding(1) var<storage, read> wq: array<u32>;
|
| 40549 |
+
@group(0) @binding(2) var<storage, read> grid: array<u32>;
|
| 40550 |
+
@group(0) @binding(3) var<storage, read_write> y: array<f32>;
|
| 40551 |
+
@group(0) @binding(4) var<uniform> p: P;
|
| 40552 |
+
@group(0) @binding(5) var<storage, read> slots: array<u32>;
|
| 40553 |
+
${deqFn(type)}
|
| 40554 |
+
const R = 16u;
|
| 40555 |
+
var<workgroup> red: array<f32, 1024>;
|
| 40556 |
+
@compute @workgroup_size(64) fn main(@builtin(workgroup_id) wg: vec3<u32>, @builtin(local_invocation_id) l: vec3<u32>) {
|
| 40557 |
+
_ = grid[0];
|
| 40558 |
+
let z = wg.z; let m0 = wg.x * R; let t = l.x; let nc = p.N / 32u;
|
| 40559 |
+
let base = slots[z] * p.slotBytes; let xo = z * p.xStride;
|
| 40560 |
+
var acc: array<f32, 16>;
|
| 40561 |
+
for (var c = t; c < nc; c += 64u) {
|
| 40562 |
+
var xv: array<f32, 32>;
|
| 40563 |
+
for (var i = 0u; i < 32u; i++) { xv[i] = xa[xo + c * 32u + i]; }
|
| 40564 |
+
let bo = (c / ${CPB}u) * ${gpuBlock(type)}u; let s = c % ${CPB}u;
|
| 40565 |
+
for (var r = 0u; r < R; r++) {
|
| 40566 |
+
let B = base + min(m0 + r, p.M - 1u) * p.rowBytes + bo;
|
| 40567 |
+
var a = 0.0;
|
| 40568 |
+
for (var g = 0u; g < 4u; g++) {
|
| 40569 |
+
let v = deq8(B, s, g);
|
| 40570 |
+
for (var j = 0u; j < 8u; j++) { a += v[j] * xv[8u * g + j]; }
|
| 40571 |
+
}
|
| 40572 |
+
acc[r] += a;
|
| 40573 |
+
}
|
| 40574 |
+
}
|
| 40575 |
+
for (var r = 0u; r < R; r++) { red[r * 64u + t] = acc[r]; }
|
| 40576 |
+
workgroupBarrier();
|
| 40577 |
+
for (var st = 32u; st > 0u; st >>= 1u) {
|
| 40578 |
+
if (t < st) { for (var r = 0u; r < R; r++) { red[r * 64u + t] += red[r * 64u + t + st]; } }
|
| 40579 |
+
workgroupBarrier();
|
| 40580 |
+
}
|
| 40581 |
+
if (t < R && m0 + t < p.M) { y[z * p.M + m0 + t] = red[t * 64u]; }
|
| 40582 |
+
}`;
|
| 40583 |
+
}
|
| 40584 |
+
|
| 40585 |
+
// Whole tensor β f32 (W_k_b, W_v_b at upload): thread i writes values 8i..8i+7 of the flat block sequence.
|
| 40586 |
+
function deqAllKernel(type) {
|
| 40587 |
+
const CPB = QTYPES[type].qk / 32;
|
| 40588 |
+
return `
|
| 40589 |
+
struct P { n8: u32 }
|
| 40590 |
+
@group(0) @binding(0) var<storage, read> wq: array<u32>;
|
| 40591 |
+
@group(0) @binding(1) var<storage, read> grid: array<u32>;
|
| 40592 |
+
@group(0) @binding(2) var<storage, read_write> y: array<f32>;
|
| 40593 |
+
@group(0) @binding(3) var<uniform> p: P;
|
| 40594 |
+
${deqFn(type)}
|
| 40595 |
+
@compute @workgroup_size(256) fn main(@builtin(workgroup_id) wg: vec3<u32>, @builtin(num_workgroups) ng: vec3<u32>, @builtin(local_invocation_id) l: vec3<u32>) {
|
| 40596 |
+
_ = grid[0];
|
| 40597 |
+
let i = (wg.y * ng.x + wg.x) * 256u + l.x; if (i >= p.n8) { return; }
|
| 40598 |
+
let c = i >> 2u; let g = i & 3u;
|
| 40599 |
+
let v = deq8((c / ${CPB}u) * ${gpuBlock(type)}u, c % ${CPB}u, g);
|
| 40600 |
+
for (var j = 0u; j < 8u; j++) { y[i * 8u + j] = v[j]; }
|
| 40601 |
+
}`;
|
| 40602 |
+
}
|
| 40603 |
+
|
| 40604 |
+
const TYPES = ['Q4_K', 'Q6_K', 'Q5_0'];
|
| 40605 |
+
|
| 40606 |
+
export class LingMoeSsd extends Qwen3MoeSsd {
|
| 40607 |
+
static configFrom(kv) { return configFromGgufLing(kv); }
|
| 40608 |
+
static tokenizerFrom(kv) { return new LingTokenizer(kv); }
|
| 40609 |
+
|
| 40610 |
+
// Reads part 2's tensor table first, then ingests both parts as one file (splitFetch) with a plan over all tensors.
|
| 40611 |
+
// opts.url alone: one GGUF (a truncated test file), ingested as it is. opts.urls: the parts from elsewhere.
|
| 40612 |
+
static async load(modelId = null, opts = {}) {
|
| 40613 |
+
const src = opts.source || LING_FLASH;
|
| 40614 |
+
const baseFetch = opts.fetch || ((u, i) => fetch(u, i));
|
| 40615 |
+
if (opts.url && !opts.urls) {
|
| 40616 |
+
const One = class extends this { static get ingestPlan() { return { layout: planLayoutLing, units: planUnitsLing }; } };
|
| 40617 |
+
return Qwen3MoeSsd.load.call(One, modelId, { ...opts, source: { repo: src.repo, file: opts.url.split('/').pop(), revision: src.revision } });
|
| 40618 |
+
}
|
| 40619 |
+
const urls = opts.urls || src.parts.map((p) => `https://huggingface.co/${src.repo}/resolve/${src.revision}/${p.file}`);
|
| 40620 |
+
const sizes = src.parts.map((p) => p.size);
|
| 40621 |
+
const fetchFn = splitFetch(urls, sizes, baseFetch);
|
| 40622 |
+
let extra = null, partErr = null;
|
| 40623 |
+
const all = (gguf) => { if (!extra) throw partErr || new Error('Ling: part 2 header not read'); return { ...gguf, tensors: [...gguf.tensors, ...extra] }; };
|
| 40624 |
+
const plan = {
|
| 40625 |
+
layout: (gguf) => planLayoutLing(all(gguf)),
|
| 40626 |
+
units: (gguf, layout) => planUnitsLing(all(gguf), layout),
|
| 40627 |
+
};
|
| 40628 |
+
const self = this;
|
| 40629 |
+
const Sub = class extends self {
|
| 40630 |
+
static get ingestPlan() { return plan; }
|
| 40631 |
+
static async open(id, o) {
|
| 40632 |
+
// Part 2's tensors are needed only to ingest, so a complete store loads without reading either part's header
|
| 40633 |
+
// (20 MB from Hugging Face otherwise). Offline, a failed read throws only if an ingest is needed.
|
| 40634 |
+
const complete = !o.reingest && (await readStore(o.key, { root: o.root || STORE_ROOT, format: STORE_FORMAT })).state === 'complete';
|
| 40635 |
+
if (!extra && !complete) {
|
| 40636 |
+
try {
|
| 40637 |
+
const h = await baseFetch(urls[0], { headers: { Range: 'bytes=0-16777215' }, cache: 'no-store' });
|
| 40638 |
+
const g1 = parseGguf(new Uint8Array(await h.arrayBuffer()));
|
| 40639 |
+
extra = await readPartTensors(urls[1], sizes[0], g1.dataStart, baseFetch);
|
| 40640 |
+
} catch (e) { partErr = e; }
|
| 40641 |
+
}
|
| 40642 |
+
return super.open(id, o);
|
| 40643 |
+
}
|
| 40644 |
+
};
|
| 40645 |
+
return Qwen3MoeSsd.load.call(Sub, modelId, { ...opts, fetch: fetchFn, url: 'split:' + urls[0], key: opts.key || 'ling-3.0-flash-q4_k_m', source: { repo: src.repo, file: src.file, revision: src.revision, size: src.size } });
|
| 40646 |
+
}
|
| 40647 |
+
|
| 40648 |
+
// Prefetch off by default: with 512 experts a guess from the layer before misses too often. Measured 2026-10-23 (M4 Pro,
|
| 40649 |
+
// 4 GB pool, 371-token prompt, 128 tokens): off 5.53 tok/s, lookahead 1 4.49, lookahead 2 3.77.
|
| 40650 |
+
constructor(device, manifest, gguf, opts) { super(device, manifest, gguf, opts); this.prefetch = opts.prefetch ?? false; }
|
| 40651 |
+
|
| 40652 |
+
chatPrompt(messages, opts) { return lingChatPrompt(messages, opts); }
|
| 40653 |
+
get stopTokenIds() { return [this.cfg.eos, this.tokenizer.ids.get('<|endoftext|>')].filter((t) => t !== undefined); }
|
| 40654 |
+
get thinkOpenTokenId() { return this.tokenizer.ids.get('<think>') ?? null; }
|
| 40655 |
+
get thinkCloseTokenId() { return this.tokenizer.ids.get('</think>') ?? null; }
|
| 40656 |
+
get thinkInPrompt() { return true; }
|
| 40657 |
+
get alwaysThinks() { return false; }
|
| 40658 |
+
get thinkPreamble() { return ''; }
|
| 40659 |
+
|
| 40660 |
+
get kernels() {
|
| 40661 |
+
if (!this._kernels) {
|
| 40662 |
+
const k = { rmsnorm: super.kernels.rmsnorm, matmulF32: super.kernels.matmulF32, argmax: super.kernels.argmax, ...LK };
|
| 40663 |
+
for (const t of TYPES) { k[`mv_${t}`] = mvKernel(t); k[`ex_${t}`] = expertKernel(t); k[`deq_${t}`] = deqAllKernel(t); }
|
| 40664 |
+
k.embed = embedKernel('Q4_K');
|
| 40665 |
+
this._kernels = k;
|
| 40666 |
+
}
|
| 40667 |
+
return this._kernels;
|
| 40668 |
+
}
|
| 40669 |
+
|
| 40670 |
+
// The MLA cache (f32 [c | k_pe] per token on 7 layers) and the KDA state (f32 128Γ128 per head, 35 layers).
|
| 40671 |
+
kvBytes() {
|
| 40672 |
+
const c = this.cfg, D = c.kvRank + c.nRot, nMla = c.mla.filter(Boolean).length, nKda = c.modelLayers - nMla;
|
| 40673 |
+
return nMla * this.maxCtx * D * 4 + nKda * (c.heads * c.kdaHd * c.kdaHd + 3 * 3 * c.heads * c.kdaHd) * 4;
|
| 40674 |
+
}
|
| 40675 |
+
|
| 40676 |
+
// Dense raw copies to the GPU, then W_k_b and W_v_b dequantized to f32 once (they serve per-head matvecs).
|
| 40677 |
+
async uploadDense(onProgress) {
|
| 40678 |
+
await super.uploadDense(onProgress);
|
| 40679 |
+
this.grid = this.buffer(gridData().byteLength, STORAGE | COPY_DST, 'dense');
|
| 40680 |
+
this.device.queue.writeBuffer(this.grid, 0, gridData());
|
| 40681 |
+
const enc = this.device.createCommandEncoder(), pass = enc.beginComputePass();
|
| 40682 |
+
this.f32 = {};
|
| 40683 |
+
for (const [name, d] of Object.entries(this.manifest.dense.tensors)) {
|
| 40684 |
+
if (!/attn_(k|v)_b\.weight$/.test(name)) continue;
|
| 40685 |
+
const n = d.dims.reduce((a, b) => a * b, 1), out = this.buffer(n * 4, STORAGE, 'dense');
|
| 40686 |
+
const t = d.type, gr = this.bind(`deq_${t}`, [this.w[name].raw, this.grid, out, this.uniform([n / 8])]);
|
| 40687 |
+
this.dispatch(pass, `deq_${t}`, gr, Math.ceil(n / 8 / 256));
|
| 40688 |
+
this.f32[name] = out;
|
| 40689 |
+
}
|
| 40690 |
+
pass.end(); this.device.queue.submit([enc.finish()]);
|
| 40691 |
+
await this.device.queue.onSubmittedWorkDone();
|
| 40692 |
+
}
|
| 40693 |
+
|
| 40694 |
+
// A matvec on a dense tensor of any kept type: { name, group, x } for encode.
|
| 40695 |
+
mv(name, x, y) {
|
| 40696 |
+
const d = this.manifest.dense.tensors[name];
|
| 40697 |
+
if (!d) throw new Error(`no tensor ${name}`);
|
| 40698 |
+
const [N, M] = d.dims;
|
| 40699 |
+
if (d.type === 'f32') return { k: 'matmulF32', g: this.bind('matmulF32', [x, this.w[name].raw, y, this.uniform([M, N])]), n: M };
|
| 40700 |
+
return { k: `mv_${d.type}`, g: this.bind(`mv_${d.type}`, [x, this.w[name].raw, this.grid, y, this.uniform([M, N, rowBytes(d.type, N), 0])]), n: Math.ceil(M / 16) };
|
| 40701 |
+
}
|
| 40702 |
+
|
| 40703 |
+
initBuffers() {
|
| 40704 |
+
const c = this.cfg, H = c.hidden, K = c.topK, F = c.expertFf, hd = c.kdaHd, NK = c.heads * hd;
|
| 40705 |
+
const D = c.kvRank + c.nRot, R = c.kvRank, nope = c.qkHd - c.nRot;
|
| 40706 |
+
if (c.experts > 512 || c.topK > 32) throw new Error(`${c.experts} experts / top-${c.topK}: the router handles at most 512 / 32`);
|
| 40707 |
+
const A = (n) => this.buffer(n * 4, STORAGE | COPY_SRC | COPY_DST);
|
| 40708 |
+
const a = this.a = {
|
| 40709 |
+
x: A(H), xn: A(H), o: A(H), sh: A(H), pxn: A(H),
|
| 40710 |
+
q: A(Math.max(NK, c.heads * c.qkHd)), k: A(NK), v: A(NK), qc: A(NK), kc: A(NK), vc: A(NK), fa: A(NK), gz: A(NK), gb: A(NK + c.heads),
|
| 40711 |
+
bl: A(c.heads), att: A(Math.max(NK, c.heads * c.vHd)), att2: A(NK),
|
| 40712 |
+
kva: A(D), qa: A(c.heads * D), sc: A(c.heads * this.maxCtx), ol: A(c.heads * R), gt: A(c.heads),
|
| 40713 |
+
ffgu: A(2 * Math.max(c.denseFf, c.shexpFf)), ffact: A(Math.max(c.denseFf, c.shexpFf)),
|
| 40714 |
+
rl: A(c.experts), sel: A(2 * K), prl: A(c.experts), psel: A(64 * this.lookahead), slots: A(K),
|
| 40715 |
+
gu: A(K * 2 * F), act: A(K * F), dn: A(K * H), logits: A(c.vocab), am: A(4), rope: A(c.nRot),
|
| 40716 |
+
};
|
| 40717 |
+
this.tok = this.buffer(16, 0x40 | COPY_DST);
|
| 40718 |
+
this.rbSel = this.buffer(256 * (1 + this.lookahead), MAP_READ | COPY_DST);
|
| 40719 |
+
this.rbLogits = this.buffer(c.vocab * 4, MAP_READ | COPY_DST);
|
| 40720 |
+
this.rbArg = this.buffer(16, MAP_READ | COPY_DST);
|
| 40721 |
+
// Recurrent state and MLA caches, per model layer.
|
| 40722 |
+
this.state = []; this.convSt = []; this.cache = [];
|
| 40723 |
+
for (let l = 0; l < c.modelLayers; l++) {
|
| 40724 |
+
if (c.mla[l]) { this.cache[l] = this.buffer(this.maxCtx * D * 4, STORAGE, 'kv'); continue; }
|
| 40725 |
+
this.state[l] = this.buffer(c.heads * hd * hd * 4, STORAGE | COPY_DST, 'kv');
|
| 40726 |
+
this.convSt[l] = ['q', 'k', 'v'].map(() => this.buffer(3 * NK * 4, STORAGE | COPY_DST, 'kv'));
|
| 40727 |
+
}
|
| 40728 |
+
const W = this.w, P = (l, n) => `blk.${l}.${n}`, raw = (n) => W[n].raw;
|
| 40729 |
+
const u = {
|
| 40730 |
+
rmsH: this.uniform([H, { f: c.eps }]), nH: this.uniform([H]), nNK: this.uniform([NK]),
|
| 40731 |
+
gates: this.uniform([NK, hd, c.heads, { f: c.gateLow }]), kda: this.uniform([hd, NK, { f: c.eps }, { f: 1 / Math.sqrt(hd) }]),
|
| 40732 |
+
hng: this.uniform([hd, { f: c.eps }]), kv: this.uniform([R, c.nRot, { f: c.eps }, 0]),
|
| 40733 |
+
mq: this.uniform([c.heads, R, nope, c.nRot]), ms: this.uniform([c.heads, D, this.maxCtx, { f: 1 / Math.sqrt(c.qkHd) }]),
|
| 40734 |
+
soft: this.uniform([this.maxCtx]), mix: this.uniform([c.heads, R, D, this.maxCtx]), mo: this.uniform([c.heads, R, c.vHd, 0]),
|
| 40735 |
+
route: this.uniform([c.experts, K, c.groups, c.groupsUsed, { f: c.expScale }, 0, 0, 0]),
|
| 40736 |
+
acc: this.uniform([H, K]), am: this.uniform([c.vocab]),
|
| 40737 |
+
};
|
| 40738 |
+
this.u = u;
|
| 40739 |
+
const swi = (Fn, k, L) => this.uniform([Fn, k, { f: L || 0 }, 0]);
|
| 40740 |
+
const embd = this.manifest.dense.tensors['token_embd.weight'];
|
| 40741 |
+
this.g = { embed: this.bind('embed', [raw('token_embd.weight'), this.grid, a.x, this.tok, this.uniform([H, rowBytes(embd.type, H)])]) };
|
| 40742 |
+
this.trunk = [];
|
| 40743 |
+
for (let l = 0; l < c.modelLayers; l++) {
|
| 40744 |
+
const t = { rmsA: this.bind('rmsnorm', [a.x, raw(P(l, 'attn_norm.weight')), a.xn, u.rmsH]), mla: c.mla[l] };
|
| 40745 |
+
if (c.mla[l]) {
|
| 40746 |
+
t.q = this.mv(P(l, 'attn_q.weight'), a.xn, a.q);
|
| 40747 |
+
t.kva = this.mv(P(l, 'attn_kv_a_mqa.weight'), a.xn, a.kva);
|
| 40748 |
+
t.gate = this.mv(P(l, 'attn_gate.weight'), a.xn, a.gt);
|
| 40749 |
+
t.kv = this.bind('mlaKv', [a.kva, raw(P(l, 'attn_kv_a_norm.weight')), a.rope, this.cache[l], this.tok, u.kv]);
|
| 40750 |
+
t.mq = this.bind('mlaQ', [a.q, this.f32[P(l, 'attn_k_b.weight')], a.rope, a.qa, u.mq]);
|
| 40751 |
+
t.score = this.bind('mlaScore', [a.qa, this.cache[l], a.sc, this.tok, u.ms]);
|
| 40752 |
+
t.soft = this.bind('softmaxS', [a.sc, this.tok, u.soft]);
|
| 40753 |
+
t.mix = this.bind('mlaMix', [a.sc, this.cache[l], a.ol, this.tok, u.mix]);
|
| 40754 |
+
t.mo = this.bind('mlaOut', [a.ol, this.f32[P(l, 'attn_v_b.weight')], a.gt, a.att, u.mo]);
|
| 40755 |
+
} else {
|
| 40756 |
+
t.q = this.mv(P(l, 'attn_q.weight'), a.xn, a.q); t.k = this.mv(P(l, 'attn_k.weight'), a.xn, a.k); t.v = this.mv(P(l, 'attn_v.weight'), a.xn, a.v);
|
| 40757 |
+
t.fa = this.mv(P(l, 'ssm_f_a.weight'), a.xn, a.fa); t.gz = this.mv(P(l, 'ssm_g_a.weight'), a.xn, a.gz); t.bl = this.mv(P(l, 'ssm_beta.weight'), a.xn, a.bl);
|
| 40758 |
+
t.conv = [['q', a.q, a.qc], ['k', a.k, a.kc], ['v', a.v, a.vc]].map(([n, x, y], i) =>
|
| 40759 |
+
this.bind('convStep', [x, raw(P(l, `ssm_conv1d_${n}.weight`)), this.convSt[l][i], y, u.nNK]));
|
| 40760 |
+
t.gates = this.bind('kdaGates', [a.fa, raw(P(l, 'ssm_dt.bias')), raw(P(l, 'ssm_a')), a.bl, a.gb, u.gates]);
|
| 40761 |
+
t.delta = this.bind('kdaStep', [a.qc, a.kc, a.vc, a.gb, this.state[l], a.att2, u.kda]);
|
| 40762 |
+
t.hng = this.bind('headNormGate', [a.att2, raw(P(l, 'ssm_norm.weight')), a.gz, a.att, u.hng]);
|
| 40763 |
+
}
|
| 40764 |
+
t.o = this.mv(P(l, 'attn_output.weight'), a.att, a.o);
|
| 40765 |
+
t.addO = this.bind('addInPlace', [a.x, a.o, u.nH]);
|
| 40766 |
+
t.rmsF = this.bind('rmsnorm', [a.x, raw(P(l, 'ffn_norm.weight')), a.xn, u.rmsH]);
|
| 40767 |
+
const ffn = (gate, up, down, Fn, L) => ({
|
| 40768 |
+
g: this.mv(P(l, gate), a.xn, a.ffgu), u: this.muOff(P(l, up), a.xn, a.ffgu, Fn),
|
| 40769 |
+
act: this.bind('swiglu', [a.ffgu, a.ffact, swi(Fn, 1, L)]), d: this.mv(P(l, down), a.ffact, a.sh), F: Fn,
|
| 40770 |
+
});
|
| 40771 |
+
if (l < c.lead) {
|
| 40772 |
+
t.ffn = ffn('ffn_gate.weight', 'ffn_up.weight', 'ffn_down.weight', c.denseFf, 0);
|
| 40773 |
+
t.addF = this.bind('addInPlace', [a.x, a.sh, u.nH]);
|
| 40774 |
+
} else {
|
| 40775 |
+
t.shexp = ffn('ffn_gate_shexp.weight', 'ffn_up_shexp.weight', 'ffn_down_shexp.weight', c.shexpFf, c.clampShexp[l]);
|
| 40776 |
+
t.router = this.mv(P(l, 'ffn_gate_inp.weight'), a.xn, a.rl);
|
| 40777 |
+
t.route = this.bind('routeLing', [a.rl, raw(P(l, 'exp_probs_b.bias')), a.sel, u.route]);
|
| 40778 |
+
}
|
| 40779 |
+
this.trunk.push(t);
|
| 40780 |
+
}
|
| 40781 |
+
// MoE layer view for the base step loop: layer l = model layer lead + l.
|
| 40782 |
+
const xp = this.manifest.experts, slotGu = xp.parts.guQ.bytes, slotDn = xp.parts.dQ.bytes, Hb = rowBytes('Q4_K', H);
|
| 40783 |
+
this.layers = [];
|
| 40784 |
+
for (let l = 0; l < c.layers; l++) {
|
| 40785 |
+
const ml = c.lead + l, dt = this.manifest.downType[l];
|
| 40786 |
+
this.layers.push({
|
| 40787 |
+
pf: Array.from({ length: Math.min(this.lookahead, c.layers - 1 - l) }, (_, i) => {
|
| 40788 |
+
const nl = ml + 1 + i;
|
| 40789 |
+
return {
|
| 40790 |
+
rms: this.bind('rmsnorm', [a.x, raw(P(nl, 'ffn_norm.weight')), a.pxn, u.rmsH]),
|
| 40791 |
+
router: this.mv(P(nl, 'ffn_gate_inp.weight'), a.pxn, a.prl),
|
| 40792 |
+
route: this.device.createBindGroup({ layout: this.pipeline('routeLing').getBindGroupLayout(0), entries: [
|
| 40793 |
+
{ binding: 0, resource: { buffer: a.prl } }, { binding: 1, resource: { buffer: raw(P(nl, 'exp_probs_b.bias')) } },
|
| 40794 |
+
{ binding: 2, resource: { buffer: a.psel, offset: 256 * i, size: 8 * K } }, { binding: 3, resource: { buffer: u.route } },
|
| 40795 |
+
] }),
|
| 40796 |
+
};
|
| 40797 |
+
}),
|
| 40798 |
+
gu: this.bind('ex_Q4_K', [a.xn, this.pool.guQ, this.grid, a.gu, this.uniform([2 * F, H, Hb, slotGu, 0, 0, 0, 0]), a.slots]),
|
| 40799 |
+
act: this.bind('swiglu', [a.gu, a.act, swi(F, K, c.clampExp[ml])]),
|
| 40800 |
+
dn: this.bind(`ex_${dt}`, [a.act, this.pool.dQ, this.grid, a.dn, this.uniform([H, F, rowBytes(dt, F), slotDn, F, 0, 0, 0]), a.slots]),
|
| 40801 |
+
dk: `ex_${dt}`,
|
| 40802 |
+
acc: this.bind('moeAccum', [a.dn, a.sel, a.sh, a.x, u.acc]),
|
| 40803 |
+
});
|
| 40804 |
+
}
|
| 40805 |
+
this.g.rmsOut = this.bind('rmsnorm', [a.x, raw('output_norm.weight'), a.xn, u.rmsH]);
|
| 40806 |
+
this.g.lm = this.mv('output.weight', a.xn, a.logits);
|
| 40807 |
+
this.g.am = this.bind('argmax', [a.logits, a.am, u.am]);
|
| 40808 |
+
// Debug: every model layer's output x for the last step (opts.captureX), read by readCapture().
|
| 40809 |
+
if (this.opts.captureX) {
|
| 40810 |
+
this.dbg = this.buffer(c.modelLayers * H * 4, STORAGE | COPY_SRC);
|
| 40811 |
+
this.dbgRead = this.buffer(c.modelLayers * H * 4, MAP_READ | COPY_DST);
|
| 40812 |
+
this.g.cap = Array.from({ length: c.modelLayers }, (_, ml) => this.bind('copyAt', [a.x, this.dbg, this.uniform([H, ml])]));
|
| 40813 |
+
}
|
| 40814 |
+
this.clearState();
|
| 40815 |
+
}
|
| 40816 |
+
|
| 40817 |
+
// The up half of a gateβup buffer: the same matvec written Fn floats in (a bind group over y at an offset).
|
| 40818 |
+
muOff(name, x, y, Fn) {
|
| 40819 |
+
const d = this.manifest.dense.tensors[name], [N, M] = d.dims;
|
| 40820 |
+
const g = this.device.createBindGroup({ layout: this.pipeline(`mv_${d.type}`).getBindGroupLayout(0), entries: [
|
| 40821 |
+
{ binding: 0, resource: { buffer: x } }, { binding: 1, resource: { buffer: this.w[name].raw } }, { binding: 2, resource: { buffer: this.grid } },
|
| 40822 |
+
{ binding: 3, resource: { buffer: y, offset: Fn * 4, size: M * 4 } }, { binding: 4, resource: { buffer: this.uniform([M, N, rowBytes(d.type, N), 0]) } },
|
| 40823 |
+
] });
|
| 40824 |
+
return { k: `mv_${d.type}`, g, n: Math.ceil(M / 16) };
|
| 40825 |
+
}
|
| 40826 |
+
|
| 40827 |
+
clearState() {
|
| 40828 |
+
const enc = this.device.createCommandEncoder();
|
| 40829 |
+
for (const s of this.state) if (s) enc.clearBuffer(s);
|
| 40830 |
+
for (const cs of this.convSt) if (cs) for (const b of cs) enc.clearBuffer(b);
|
| 40831 |
+
this.device.queue.submit([enc.finish()]);
|
| 40832 |
+
}
|
| 40833 |
+
|
| 40834 |
+
// ββ forward ββ
|
| 40835 |
+
run(pass, m) { this.dispatch(pass, m.k, m.g, m.n); }
|
| 40836 |
+
encodeFfn(pass, f) {
|
| 40837 |
+
const d = (n, g, x, y, z) => this.dispatch(pass, n, g, x, y, z);
|
| 40838 |
+
this.run(pass, f.g); this.run(pass, f.u);
|
| 40839 |
+
d('swiglu', f.act, Math.ceil(f.F / 256));
|
| 40840 |
+
this.run(pass, f.d);
|
| 40841 |
+
}
|
| 40842 |
+
// Model layer ml's attention block, then (MoE layers) its FFN norm, shared expert, router and the prefetch guesses.
|
| 40843 |
+
encodeTrunk(pass, ml, seqLen, pf) {
|
| 40844 |
+
const c = this.cfg, t = this.trunk[ml], d = (n, g, x, y, z) => this.dispatch(pass, n, g, x, y, z), H = c.hidden, NK = c.heads * c.kdaHd;
|
| 40845 |
+
d('rmsnorm', t.rmsA, 1);
|
| 40846 |
+
if (t.mla) {
|
| 40847 |
+
this.run(pass, t.q); this.run(pass, t.kva); this.run(pass, t.gate);
|
| 40848 |
+
d('mlaKv', t.kv, 1);
|
| 40849 |
+
d('mlaQ', t.mq, Math.ceil((c.kvRank + c.nRot / 2) / 256), c.heads);
|
| 40850 |
+
d('mlaScore', t.score, Math.ceil(c.heads * seqLen / 256));
|
| 40851 |
+
d('softmaxS', t.soft, c.heads);
|
| 40852 |
+
d('mlaMix', t.mix, Math.ceil(c.kvRank / 256), c.heads);
|
| 40853 |
+
d('mlaOut', t.mo, c.heads);
|
| 40854 |
+
} else {
|
| 40855 |
+
this.run(pass, t.q); this.run(pass, t.k); this.run(pass, t.v); this.run(pass, t.fa); this.run(pass, t.gz); this.run(pass, t.bl);
|
| 40856 |
+
for (const g of t.conv) d('convStep', g, Math.ceil(NK / 256));
|
| 40857 |
+
d('kdaGates', t.gates, Math.ceil(NK / 256));
|
| 40858 |
+
d('kdaStep', t.delta, c.heads);
|
| 40859 |
+
d('headNormGate', t.hng, c.heads);
|
| 40860 |
+
}
|
| 40861 |
+
this.run(pass, t.o);
|
| 40862 |
+
d('addInPlace', t.addO, Math.ceil(H / 256));
|
| 40863 |
+
d('rmsnorm', t.rmsF, 1);
|
| 40864 |
+
if (t.ffn) { this.encodeFfn(pass, t.ffn); d('addInPlace', t.addF, Math.ceil(H / 256)); if (this.g.cap) d('copyAt', this.g.cap[ml], Math.ceil(H / 256)); return; }
|
| 40865 |
+
if (this.prefetch && pf) for (const p of pf) { d('rmsnorm', p.rms, 1); this.run(pass, p.router); d('routeLing', p.route, 1); }
|
| 40866 |
+
this.encodeFfn(pass, t.shexp);
|
| 40867 |
+
this.run(pass, t.router);
|
| 40868 |
+
d('routeLing', t.route, 1);
|
| 40869 |
+
}
|
| 40870 |
+
// The base loop's attention hook for MoE layer l: the dense lead layers go first with layer 0.
|
| 40871 |
+
encodeAttention(pass, l, seqLen) {
|
| 40872 |
+
if (l === 0) for (let ml = 0; ml < this.cfg.lead; ml++) this.encodeTrunk(pass, ml, seqLen, null);
|
| 40873 |
+
this.encodeTrunk(pass, this.cfg.lead + l, seqLen, this.layers[l].pf);
|
| 40874 |
+
}
|
| 40875 |
+
encodeExperts(pass, l) {
|
| 40876 |
+
const c = this.cfg, g = this.layers[l], d = (n, gr, x, y, z) => this.dispatch(pass, n, gr, x, y, z);
|
| 40877 |
+
d('ex_Q4_K', g.gu, Math.ceil(2 * c.expertFf / 16), 1, c.topK);
|
| 40878 |
+
d('swiglu', g.act, Math.ceil(c.topK * c.expertFf / 256));
|
| 40879 |
+
d(g.dk, g.dn, Math.ceil(c.hidden / 16), 1, c.topK);
|
| 40880 |
+
d('moeAccum', g.acc, Math.ceil(c.hidden / 256));
|
| 40881 |
+
if (this.g.cap) d('copyAt', this.g.cap[c.lead + l], Math.ceil(c.hidden / 256));
|
| 40882 |
+
}
|
| 40883 |
+
async readCapture() {
|
| 40884 |
+
const enc = this.device.createCommandEncoder();
|
| 40885 |
+
enc.copyBufferToBuffer(this.dbg, 0, this.dbgRead, 0, this.dbg.size);
|
| 40886 |
+
this.device.queue.submit([enc.finish()]);
|
| 40887 |
+
await this.dbgRead.mapAsync(1);
|
| 40888 |
+
const out = new Float32Array(this.dbgRead.getMappedRange().slice(0));
|
| 40889 |
+
this.dbgRead.unmap();
|
| 40890 |
+
return out;
|
| 40891 |
+
}
|
| 40892 |
+
encodeEmbed(pass) { this.dispatch(pass, 'embed', this.g.embed, Math.ceil(this.cfg.hidden / 8 / 256)); }
|
| 40893 |
+
encodeHead(pass, want) {
|
| 40894 |
+
this.dispatch(pass, 'rmsnorm', this.g.rmsOut, 1);
|
| 40895 |
+
this.run(pass, this.g.lm);
|
| 40896 |
+
if (want === 'argmax') this.dispatch(pass, 'argmax', this.g.am, 1);
|
| 40897 |
+
}
|
| 40898 |
+
|
| 40899 |
+
writeTokenUniforms(token, pos) {
|
| 40900 |
+
const c = this.cfg;
|
| 40901 |
+
this.device.queue.writeBuffer(this.tok, 0, new Uint32Array([token, pos, pos + 1, 0]));
|
| 40902 |
+
// GGML NORM RoPE angles: pair i turns by pos Β· base^(-2i/nRot); cos then sin.
|
| 40903 |
+
const half = c.nRot / 2, r = new Float32Array(c.nRot), inv = Math.fround(-1 / c.nRot);
|
| 40904 |
+
for (let i = 0; i < half; i++) {
|
| 40905 |
+
const th = Math.fround(pos * Math.fround(Math.pow(c.ropeTheta, Math.fround(inv * 2 * i))));
|
| 40906 |
+
r[i] = Math.cos(th); r[half + i] = Math.sin(th);
|
| 40907 |
+
}
|
| 40908 |
+
this.device.queue.writeBuffer(this.a.rope, 0, r);
|
| 40909 |
+
}
|
| 40910 |
+
|
| 40911 |
+
reset() { super.reset(); if (this.state) this.clearState(); }
|
| 40912 |
+
// The KDA state cannot step back: a prompt that extends the cached tokens continues, any other starts over.
|
| 40913 |
+
async prefill(ids, want = 'argmax') {
|
| 40914 |
+
let common = 0;
|
| 40915 |
+
while (common < ids.length - 1 && common < this.cached.length && this.cached[common] === ids[common]) common++;
|
| 40916 |
+
if (common < this.cached.length) { this.reset(); common = 0; }
|
| 40917 |
+
let r = null;
|
| 40918 |
+
for (let i = common; i < ids.length; i++) r = await this.step(ids[i], i === ids.length - 1 ? want : 'none');
|
| 40919 |
+
return r;
|
| 40920 |
+
}
|
| 40921 |
+
}
|
| 40922 |
+
</script>
|
| 40923 |
<script type="text/plain" data-module="minicpmv_vision.js">/* minicpmv_vision.js β MiniCPM-V 4.7's vision encoder (mmproj GGUF, projector `minicpmv4_7`) on WebGPU.
|
| 40924 |
*
|
| 40925 |
* One image tile in, its token embeddings out (64 Γ 2048 for a 448 Γ 448 tile), ready to splice into the text model.
|
|
|
|
| 44600 |
};
|
| 44601 |
const FORMAT = 'localmind-qwen3moe-ssd/1';
|
| 44602 |
const OPFS_ROOT = 'localmind-ssd';
|
| 44603 |
+
// for engines built on this one that check their store before loading (ling_moe_ssd.js)
|
| 44604 |
+
export { FORMAT as STORE_FORMAT, OPFS_ROOT as STORE_ROOT };
|
| 44605 |
|
| 44606 |
// ββ GGUF ββββββββββββββββββββββββββββββββββββββββββββββββββββββββββββββββββββ
|
| 44607 |
// qwen3moe (rung 2a) and qwen35moe (Qwen3.5/3.6 MoE, rung 2b: qwen35_moe_ssd.js) share the
|