Upload qvac-gpu.js with huggingface_hub

qvac-gpu.js

@@ -1065,6 +1065,12 @@ export async function createQvacGPU(manifest, fetchTensor, cap = 64, eos = 2, st
   };
   if (!frameGran) await upW("lm_head");                    // resident in resident/layer modes; TILED (played) in frame mode
   let RQ = null, RS = null, packLayout = null, packStride = 0, packQbytes = 0;
+  // DRAFT (t2 layer-streaming — needs WebGPU verification): BitNet/Falcon t2 has a SCALAR scale per matrix
+  // (baked into the matmul uniform via fbits), not a per-block scale buffer. So a t2 streamed layer packs
+  // q ONLY (no scale blob), and each layer's per-role scalar is written into the reusable R[role].uni before
+  // that layer's matmul. t2scales[l][role] = fbits(scale) is precomputed from the manifest (no fetch).
+  const t2stream = !!stream && !moe && !frameGran && ROLES.every((r) => (tmap[`l0.${r}`] || {}).fmt === "t2");
+  let t2scales = null;
   let opfsStore = null;
   const frameMan = {};                                     // name → {off, qlen, slen, N, K} (frame layout)
   if (frameGran) {
@@ -1104,20 +1110,27 @@ export async function createQvacGPU(manifest, fetchTensor, cap = 64, eos = 2, st
     // The packed bytes live in JS ("layer") or on DISK in OPFS ("opfs" — the
     // RAM-wall break); EITHER way the per-layer body is the SAME proven layerBody().
     const al = (n) => Math.ceil(n / 256) * 256;
-    const qbl = (t) => Math.ceil(qlenOf(t) / 4) * 4;       // padded q length — from manifest, no fetch
+    // t2 (BitNet/Falcon): q is codes only (N·K/4 B, 4 trits/byte) + a SCALAR scale (→ uniform); no scale blob.
+    // Every other stream fmt (q3f/q4/e8/t2r): q (padded) + a per-block scale blob at soff. Same packLayer shape.
+    const qbl = (t) => t2stream ? Math.ceil((t.N * t.K / 4) / 4) * 4 : Math.ceil(qlenOf(t) / 4) * 4;
+    const sbl = (t) => t2stream ? 0 : slenOf(t);
     packLayout = {}; let qo = 0, so = 0;
-    for (const role of ROLES) { const t = tmap[`l0.${role}`]; qo = al(qo); so = al(so); packLayout[role] = { qoff: qo, qsize: qbl(t), soff: so, ssize: slenOf(t), N: t.N, K: t.K }; qo += qbl(t); so += slenOf(t); }
+    for (const role of ROLES) { const t = tmap[`l0.${role}`]; qo = al(qo); so = al(so); packLayout[role] = { qoff: qo, qsize: qbl(t), soff: so, ssize: sbl(t), N: t.N, K: t.K }; qo += qbl(t); so += sbl(t); }
     const packQ = al(qo), packS = al(so);
     RQ = dev.createBuffer({ size: packQ, usage: U.STORAGE | U.COPY_DST });
     RS = dev.createBuffer({ size: Math.max(16, packS), usage: U.STORAGE | U.COPY_DST });
     streamBuf = packQ + Math.max(16, packS);
+    // t2: per-layer-per-role scalar (fbits), refreshed into R[role].uni each layer in the forward. Manifest-only.
+    if (t2stream) { t2scales = []; for (let l = 0; l < n_layers; l++) { const row = {}; for (const role of ROLES) row[role] = fbits((tmap[`l${l}.${role}`] || {}).s ?? 1); t2scales.push(row); } }
     for (const role of ROLES) {
       const L = packLayout[role];
-      R[role] = { qbuf: { buffer: RQ, offset: L.qoff, size: L.qsize }, sbuf: { buffer: RS, offset: L.soff, size: L.ssize }, uni: ubuf(new Uint32Array([L.K, L.N, L.K / 32, 0])), N: L.N };
+      R[role] = t2stream
+        ? { qbuf: { buffer: RQ, offset: L.qoff, size: L.qsize }, uni: ubuf(new Uint32Array([L.K, L.N, L.K / 16, 0])), N: L.N, K: L.K, t2: true }   // scale.w set per layer
+        : { qbuf: { buffer: RQ, offset: L.qoff, size: L.qsize }, sbuf: { buffer: RS, offset: L.soff, size: L.ssize }, uni: ubuf(new Uint32Array([L.K, L.N, L.K / 32, 0])), N: L.N };
     }
-    const packLayer = async (l) => {                       // build one layer's packed (q,s) from the GGUF
+    const packLayer = async (l) => {                       // build one layer's packed (q[,s]) from the κ-object
       const q = new Uint8Array(packQ), s = new Uint8Array(packS);
-      for (const role of ROLES) { const p = await parts(`l${l}.${role}`); q.set(p.q, packLayout[role].qoff); s.set(new Uint8Array(p.s.buffer, p.s.byteOffset, p.s.byteLength), packLayout[role].soff); }
+      for (const role of ROLES) { const p = await parts(`l${l}.${role}`); q.set(p.q, packLayout[role].qoff); if (!t2stream) s.set(new Uint8Array(p.s.buffer, p.s.byteOffset, p.s.byteLength), packLayout[role].soff); }
       return { q, s };
     };
     if (opfs) {                                            // DISK-backed: page each layer's packed blob in per token
@@ -1651,6 +1664,8 @@ export async function createQvacGPU(manifest, fetchTensor, cap = 64, eos = 2, st
           dev.queue.writeBuffer(RQ, 0, Wb[l].q);           // one fat DMA for the whole layer's matrices
           dev.queue.writeBuffer(RS, 0, Wb[l].s);           // + one for all its scales
         }
+        // t2: this layer's per-role scalar scales into the reused uniforms (matmul reads scale from uni.w).
+        if (t2stream) for (const role of ROLES) { const L = packLayout[role]; dev.queue.writeBuffer(R[role].uni, 0, new Uint32Array([L.K, L.N, L.K / 16, t2scales[l][role]])); }
         const enc = dev.createCommandEncoder();
         cur = layerBody(enc, l, cur, (role) => R[role]);
         dev.queue.submit([enc.finish()]);