Spaces:
Running
Running
decode(): report live-path GPU ms/tok for clean CPU/GPU split
Browse files- qvac-gpu.js +2 -0
qvac-gpu.js
CHANGED
|
@@ -1804,9 +1804,11 @@ export async function createQvacGPU(manifest, fetchTensor, cap = 64, eos = 2, st
|
|
| 1804 |
mmW(enc, B.normed, W["lm_head"], B.logits);
|
| 1805 |
}
|
| 1806 |
enc.copyBufferToBuffer(DEC.ring, base * 4, DEC.stg, 0, n * 4);
|
|
|
|
| 1807 |
dev.queue.submit([enc.finish()]);
|
| 1808 |
pos = pos0 + n;
|
| 1809 |
await DEC.stg.mapAsync(GPUMapMode.READ);
|
|
|
|
| 1810 |
const win = new Uint32Array(DEC.stg.getMappedRange().slice(0)).subarray(0, n); DEC.stg.unmap();
|
| 1811 |
for (let k = 0; k < n; k++) {
|
| 1812 |
if (win[k] === eos) { pos = pos0 + k; done = true; break; } // trim + rewind: stale KV beyond pos is never read
|
|
|
|
| 1804 |
mmW(enc, B.normed, W["lm_head"], B.logits);
|
| 1805 |
}
|
| 1806 |
enc.copyBufferToBuffer(DEC.ring, base * 4, DEC.stg, 0, n * 4);
|
| 1807 |
+
const _tSub = performance.now();
|
| 1808 |
dev.queue.submit([enc.finish()]);
|
| 1809 |
pos = pos0 + n;
|
| 1810 |
await DEC.stg.mapAsync(GPUMapMode.READ);
|
| 1811 |
+
timing = { encode: 0, exec: (performance.now() - _tSub) / n }; // GPU+fence ms/token on the live decode path (diagnostics)
|
| 1812 |
const win = new Uint32Array(DEC.stg.getMappedRange().slice(0)).subarray(0, n); DEC.stg.unmap();
|
| 1813 |
for (let k = 0; k < n; k++) {
|
| 1814 |
if (win[k] === eos) { pos = pos0 + k; done = true; break; } // trim + rewind: stale KV beyond pos is never read
|