| |
| |
| |
| import { Engine } from './model.js'; |
| import { Tracker, toGray } from './tracker.js'; |
|
|
| |
| |
| const WEIGHTS = new URL('../weights/falcon-q8-lm4.bin', import.meta.url).href; |
| const TRANSFORMERS = '/assets/transformers.web-DaZ4EEMB.js'; |
| const HF_ID = 'onnx-community/falcon-perception-onnx-webgpu'; |
| const FALLBACK_TOKENS = [37462, 1978, 20528, 821, 790, 6883, 549, 264, 258]; |
|
|
| const CFG = { |
| patch: 16, dim: 1024, coordTok: 240, sizeTok: 241, segTok: 262, eos: 11, |
| imgId: 227, clsId: 244, imgEnd: 230, presence: 268, absence: 269, |
| endQuery: 263, ropeTheta: 1e4, maxSteps: 510, L: 1024, |
| }; |
| const REGISTERS = [245, 246, 247, 248]; |
| const MAX_S = 1088, MAX_T = 1664; |
|
|
| let eng = null, tokenizer = null, busy = false, queued = null; |
| let tracker = null; |
| let panDX = 0, panDY = 0; |
| const post = (m) => self.postMessage(m); |
|
|
| |
| |
| |
| |
| |
| |
| |
| |
| |
| |
| |
| |
| |
| |
| |
| |
| |
| |
| |
| const OPTS = { spec: true, specTol: 6, |
| gate: false, gateMaxMs: 2500, gateDiffThr: 0.015, |
| maxVideoSteps: 120, videoRes: 256, decodeCap: 0 }; |
|
|
| |
| |
| let lastEmission = null; |
| let lastKey = null; |
| let specBias = [0, 0]; |
| |
| |
| let specHist = []; |
| let specSkips = 0; |
| |
| |
| |
|
|
| |
|
|
| async function fetchProgress(url, label, sizeHint) { |
| const r = await fetch(url); |
| if (!r.ok) throw new Error(`fetch ${label}: ${r.status}`); |
| const total = +r.headers.get('content-length') || sizeHint || 0; |
| const reader = r.body.getReader(); |
| const chunks = []; let loaded = 0; |
| for (;;) { |
| const { done, value } = await reader.read(); |
| if (done) break; |
| chunks.push(value); loaded += value.length; |
| post({ type: 'progress', progress: Math.round(loaded / (total || loaded) * 100), |
| loaded, total, file: label }); |
| } |
| const out = new Uint8Array(loaded); let o = 0; |
| for (const c of chunks) { out.set(c, o); o += c.length; } |
| return out.buffer; |
| } |
|
|
| async function load(opts = {}) { |
| try { |
| const transformersUrl = opts.transformersUrl || TRANSFORMERS; |
| const weightsUrl = opts.weightsUrl || WEIGHTS; |
| const tokP = import(transformersUrl) |
| .then(({ AutoTokenizer }) => AutoTokenizer.from_pretrained(HF_ID)) |
| .then((t) => { tokenizer = t; console.log('tokenizer loaded'); }) |
| .catch((e) => console.warn('tokenizer fallback:', e.message)); |
| |
| |
| |
| const probe = await (navigator.gpu?.requestAdapter({ powerPreference: 'high-performance' }) |
| ?? Promise.resolve(null)); |
| if (!probe) throw new Error('нет WebGPU — нужен Chrome/Edge 125+'); |
| for (const f of ['shader-f16', 'subgroups']) |
| if (!probe.features.has(f)) throw new Error(`нет WebGPU-фичи ${f}`); |
| const wbuf = await fetchProgress(weightsUrl, 'falcon-q8-lm4.bin', 724248832); |
| const adapter = await navigator.gpu.requestAdapter({ powerPreference: 'high-performance' }); |
| const feats = ['shader-f16', 'subgroups']; |
| if (adapter.features.has('chromium-experimental-subgroup-matrix')) |
| feats.push('chromium-experimental-subgroup-matrix'); |
| const device = await adapter.requestDevice({ |
| requiredFeatures: feats, |
| requiredLimits: { |
| maxStorageBufferBindingSize: Math.min(1 << 30, adapter.limits.maxStorageBufferBindingSize), |
| maxBufferSize: Math.min(1 << 30, adapter.limits.maxBufferSize), |
| }, |
| }); |
| device.addEventListener('uncapturederror', (e) => console.error('WebGPU:', e.error.message)); |
| |
| device.lost.then((i) => post({ type: 'error', fatal: true, message: `engine device lost: ${i.message}` })); |
| eng = await Engine.create(device, wbuf, { maxS: MAX_S, maxT: MAX_T, sgmat: false }); |
| |
| const adapter2 = await navigator.gpu.requestAdapter({ powerPreference: 'high-performance' }); |
| const tdev = await adapter2.requestDevice(); |
| tdev.lost.then((i) => post({ type: 'error', fatal: true, message: `tracker device lost: ${i.message}` })); |
| tracker = await new Tracker(tdev).init(); |
| await tokP; |
| post({ type: 'loaded' }); |
| } catch (e) { |
| post({ type: 'error', message: e.message || String(e) }); |
| } |
| } |
|
|
| |
|
|
| function preprocess(image, video = false) { |
| const p = CFG.patch; |
| let w, h; |
| if (video) { w = h = OPTS.videoRes; } |
| else { |
| const scale = Math.min(512 / image.width, 512 / image.height, 1); |
| w = Math.max(p, Math.round(image.width * scale / p) * p); |
| h = Math.max(p, Math.round(image.height * scale / p) * p); |
| } |
| const ctx = new OffscreenCanvas(w, h).getContext('2d'); |
| ctx.drawImage(image, 0, 0, w, h); |
| const imgData = ctx.getImageData(0, 0, w, h); |
| const img = imgData.data; |
| const gray = toGray(imgData, w, h); |
| const ph = h / p, pw = w / p, n = ph * pw, pl = p * p * 3; |
| const patches = new Float32Array(n * pl); |
| for (let py = 0; py < ph; py++) |
| for (let px = 0; px < pw; px++) { |
| const base = (py * pw + px) * pl; |
| for (let r = 0; r < p; r++) |
| for (let c = 0; c < p; c++) { |
| const src = ((py * p + r) * w + px * p + c) * 4; |
| const dst = base + (r * p + c) * 3; |
| patches[dst] = img[src] / 255 * 2 - 1; |
| patches[dst + 1] = img[src + 1] / 255 * 2 - 1; |
| patches[dst + 2] = img[src + 2] / 255 * 2 - 1; |
| } |
| } |
| return { patches, ph, pw, gray, w, h }; |
| } |
|
|
| function posGrid(ph, pw) { |
| const sy = Math.sqrt(pw / ph), sx = Math.sqrt(ph / pw); |
| const g = new Float32Array(ph * pw * 2); |
| for (let i = 0; i < ph; i++) { |
| const rv = ph > 1 ? sx * (2 * i / (ph - 1) - 1) : 0; |
| for (let j = 0; j < pw; j++) { |
| const cv = pw > 1 ? sy * (2 * j / (pw - 1) - 1) : 0; |
| g[(i * pw + j) * 2] = rv; |
| g[(i * pw + j) * 2 + 1] = cv; |
| } |
| } |
| return g; |
| } |
|
|
| |
|
|
| const iou = (a, b) => { |
| const w = Math.min(a.x2, b.x2) - Math.max(a.x1, b.x1); |
| const h = Math.min(a.y2, b.y2) - Math.max(a.y1, b.y1); |
| if (w <= 0 || h <= 0) return 0; |
| const ar = (r) => Math.max(0, r.x2 - r.x1) * Math.max(0, r.y2 - r.y1); |
| const i = w * h; |
| return i / (ar(a) + ar(b) - i); |
| }; |
| function nms(dets, thr = 0.6) { |
| const order = dets.map((d, i) => [((d.box.x2 - d.box.x1) * (d.box.y2 - d.box.y1)), i]) |
| .sort((a, b) => b[0] - a[0]); |
| const keepIdx = new Set(), keptBoxes = []; |
| for (const [, i] of order) { |
| if (!keptBoxes.some((k) => iou(dets[i].box, k) > thr)) { |
| keepIdx.add(i); keptBoxes.push(dets[i].box); |
| } |
| } |
| return dets.filter((_, i) => keepIdx.has(i)); |
| } |
| const argmax = (a, o = 0, n = a.length) => { |
| let bi = 0, bv = -Infinity; |
| for (let i = 0; i < n; i++) if (a[o + i] > bv) { bv = a[o + i]; bi = i; } |
| return bi; |
| }; |
|
|
| function frameGray(image, video) { |
| const fit = (d) => (video ? OPTS.videoRes : Math.max(CFG.patch, |
| Math.round(d * Math.min(512 / image.width, 512 / image.height, 1) / CFG.patch) * CFG.patch)); |
| const w = fit(image.width), h = fit(image.height); |
| const ctx = new OffscreenCanvas(w, h).getContext('2d'); |
| ctx.drawImage(image, 0, 0, w, h); |
| return { gray: toGray(ctx.getImageData(0, 0, w, h), w, h), w, h }; |
| } |
|
|
| |
| async function trackFrame(msg, accumulatePan, pre = null) { |
| const { image, query, requestId, sourceTimestampMs, capturedAtMs, video } = msg; |
| try { |
| const t0 = performance.now(); |
| const { gray, w, h } = pre || frameGray(image, video); |
| const dets = await tracker.track(gray, w, h); |
| if (accumulatePan) { |
| panDX += tracker.lastShift?.dx || 0; |
| panDY += tracker.lastShift?.dy || 0; |
| } |
| post({ type: 'result', source: 'track', detections: dets, |
| inferenceMs: performance.now() - t0, |
| query, requestId, sourceTimestampMs, capturedAtMs, tracked: true }); |
| } catch (e) { |
| console.warn('[engine] track error:', e.message); |
| } finally { image.close?.(); } |
| } |
|
|
| |
| |
| |
| |
| function gateStatic(gray, w, h, query) { |
| if (!lastKey || lastKey.w !== w || lastKey.h !== h || lastKey.query !== query) return false; |
| if (performance.now() - lastKey.time > OPTS.gateMaxMs) return false; |
| const boxes = [...lastKey.boxes, ...tracker.boxes.map((t) => t.box)].map((b) => { |
| const mx = (b.x2 - b.x1) * 0.1 + 0.01, my = (b.y2 - b.y1) * 0.1 + 0.01; |
| return { x1: (b.x1 - mx) * w, x2: (b.x2 + mx) * w, |
| y1: (b.y1 - my) * h, y2: (b.y2 + my) * h }; |
| }); |
| let sum = 0, n = 0, masked = 0, total = 0; |
| for (let y = 2; y < h; y += 4) |
| for (let x = 2; x < w; x += 4) { |
| total++; |
| if (boxes.some((b) => x >= b.x1 && x <= b.x2 && y >= b.y1 && y <= b.y2)) { masked++; continue; } |
| sum += Math.abs(gray[y * w + x] - lastKey.gray[y * w + x]); n++; |
| } |
| if (n < 256) return false; |
| return sum / n < OPTS.gateDiffThr; |
| } |
|
|
| async function detect(msg) { |
| const { image, query, video } = msg; |
| if (busy) { await trackFrame(msg, true); return; } |
| if (video && OPTS.gate && tracker) { |
| try { |
| const pre = frameGray(image, true); |
| if (gateStatic(pre.gray, pre.w, pre.h, query)) { |
| await trackFrame(msg, false, pre); |
| return; |
| } |
| } catch (e) { console.warn('[engine] gate error:', e.message); } |
| } |
| busy = true; |
| panDX = 0; panDY = 0; |
| await heavyDetect(msg); |
| } |
|
|
| async function heavyDetect(msg) { |
| const { image, query, requestId, sourceTimestampMs, capturedAtMs, video } = msg; |
| const reply = (dets, ms, stats) => post({ type: 'result', source: 'detect', detections: dets, |
| inferenceMs: ms, query, requestId, sourceTimestampMs, capturedAtMs, stats }); |
| try { |
| const t0 = performance.now(); |
| const { patches, ph, pw, gray, w: fw, h: fh } = preprocess(image, video); |
| const nP = ph * pw; |
| let text = FALLBACK_TOKENS; |
| if (tokenizer) { |
| text = Array.from(tokenizer.encode( |
| `Segment these expressions in the image:<|start_of_query|>${query}<|REF_SEG|>`, |
| { add_special_tokens: false })); |
| } |
| const imgStart = 1 + REGISTERS.length; |
| const seq = [CFG.clsId, ...REGISTERS, |
| ...Array(nP).fill(CFG.imgId), CFG.imgEnd, ...text]; |
| const C = seq.length; |
| const bidirEnd = imgStart + nP + 1; |
| if (C > MAX_S) throw new Error(`seq_len ${C} > maxS ${MAX_S}`); |
|
|
| |
| const skip = new Set([...REGISTERS, CFG.imgId, CFG.imgEnd]); |
| const pos = new Float32Array(C); |
| let acc = 0; |
| for (let i = 0; i < C; i++) { acc += (skip.has(seq[i]) && i < bidirEnd) ? 0 : 1; pos[i] = acc - 1; } |
| const r1 = new Float32Array(MAX_T * 32 * 2); |
| for (let t = 0; t < MAX_T; t++) { |
| const P = t < C ? pos[t] : pos[C - 1] + 1 + (t - C); |
| for (let f = 0; f < 32; f++) { |
| const a = P / CFG.ropeTheta ** (2 * f / 64); |
| r1[(t * 32 + f) * 2] = Math.cos(a); |
| r1[(t * 32 + f) * 2 + 1] = Math.sin(a); |
| } |
| } |
| const r2 = new Float32Array(MAX_T * 16 * 32 * 2); |
| for (let i = 0; i < MAX_T * 16 * 32; i++) r2[i * 2] = 1; |
| const grid = posGrid(ph, pw); |
| const golden = eng.golden; |
| for (let k = 0; k < nP; k++) { |
| const s = imgStart + k, p0 = grid[k * 2], p1 = grid[k * 2 + 1]; |
| for (let hh = 0; hh < 16; hh++) |
| for (let f = 0; f < 32; f++) { |
| const th = p0 * golden[(hh * 32 + f) * 2] + p1 * golden[(hh * 32 + f) * 2 + 1]; |
| const o = ((s * 16 + hh) * 32 + f) * 2; |
| r2[o] = Math.cos(th); r2[o + 1] = Math.sin(th); |
| } |
| } |
| eng.setRopeTables(r1, r2); |
|
|
| |
| const tp0 = performance.now(); |
| await eng.buildPrefillEmbeddings(new Uint32Array(seq), patches, imgStart); |
| await eng.prefill(null, bidirEnd, {}, C); |
| let logits = await eng.readF32(eng.b.logits, 65536); |
| const tPrefill = performance.now() - tp0; |
| let tHeads = 0, tSteps = 0, nSteps = 0, tSpec = 0, nSpecOk = 0, nSpecRun = 0; |
| const presence = logits[CFG.presence], absence = logits[CFG.absence]; |
| console.log(`[engine] prefill ${C} ток: presence=${presence.toFixed(2)} absence=${absence.toFixed(2)}`); |
| const score = 1 / (1 + Math.exp(-(presence - absence))); |
| const dets = []; |
| const presStats = { presence: +presence.toFixed(2), absence: +absence.toFixed(2), score: +score.toFixed(3) }; |
| if (presence <= absence && presence < 0) { |
| reply([], performance.now() - t0, { ...presStats, gatedOut: true, prefillMs: Math.round(tPrefill) }); |
| busy = false; runQueued(image); return; |
| } |
|
|
| |
| const STOP = new Set([CFG.eos, CFG.endQuery]); |
| const emission = []; |
| let terminal = null; |
|
|
| const mnL = Math.log2(1 / CFG.L); |
| const coordFromBins = (bi, bj) => [bi / (CFG.L - 1), bj / (CFG.L - 1)]; |
| const sizeFromBins = (bi, bj) => |
| [2 ** (bi / (CFG.L - 1) * -mnL + mnL), 2 ** (bj / (CFG.L - 1) * -mnL + mnL)]; |
| |
| const coordBinsDedup = (hl) => { |
| let bi, bj; |
| for (let tr = 0; tr < 100; tr++) { |
| bi = argmax(hl, 0, CFG.L); bj = argmax(hl, CFG.L, CFG.L); |
| const cx = bi / (CFG.L - 1), cy = bj / (CFG.L - 1); |
| if (!dets.some((d) => Math.abs((d.box.x1 + d.box.x2) / 2 - cx) < 0.01 && |
| Math.abs((d.box.y1 + d.box.y2) / 2 - cy) < 0.01)) break; |
| hl[bi] = -Infinity; hl[CFG.L + bj] = -Infinity; |
| } |
| return [bi, bj]; |
| }; |
| const pushDet = (cBins, sBins) => { |
| const coord = coordFromBins(...cBins), size = sizeFromBins(...sBins); |
| dets.push({ label: query, score, |
| box: { x1: Math.max(0, coord[0] - size[1] / 2), y1: Math.max(0, coord[1] - size[0] / 2), |
| x2: Math.min(1, coord[0] + size[1] / 2), y2: Math.min(1, coord[1] + size[0] / 2) }, |
| mask: null }); |
| post({ type: 'result', source: 'detect', detections: [...dets], |
| inferenceMs: performance.now() - t0, |
| query, requestId, sourceTimestampMs, capturedAtMs, partial: true }); |
| }; |
|
|
| |
| |
| |
| const seqDecode = async (nxt, preHead) => { |
| let cBins = null, sBins = null; |
| let stepCap = video ? Math.min(CFG.maxSteps, OPTS.maxVideoSteps) : CFG.maxSteps; |
| if (OPTS.decodeCap > 0) stepCap = Math.min(stepCap, OPTS.decodeCap); |
| for (let st = 0; st < stepCap && eng.T < MAX_T - 1; st++) { |
| if (st === stepCap - 1) console.warn(`[engine] декод упёрся в потолок ${stepCap} шагов — обрыв`); |
| if (STOP.has(nxt) || nxt === CFG.absence) { terminal = nxt; return; } |
| let embNext = null, entry = { id: nxt, kind: 'g' }; |
| if (nxt === CFG.coordTok) { |
| let hl; |
| if (preHead) { hl = preHead.coord; preHead = null; } |
| else { |
| const th0 = performance.now(); |
| hl = await eng.runHead('coord'); |
| tHeads += performance.now() - th0; |
| } |
| cBins = coordBinsDedup(hl); |
| embNext = eng.encodeCoords('coord', coordFromBins(...cBins)); |
| entry = { id: nxt, kind: 'c', bins: cBins }; |
| } else if (nxt === CFG.sizeTok) { |
| let hl; |
| if (preHead) { hl = preHead.size; preHead = null; } |
| else { |
| const th0 = performance.now(); |
| hl = await eng.runHead('size'); |
| tHeads += performance.now() - th0; |
| } |
| sBins = [argmax(hl, 0, CFG.L), argmax(hl, CFG.L, CFG.L)]; |
| embNext = eng.encodeCoords('size', sizeFromBins(...sBins)); |
| entry = { id: nxt, kind: 's', bins: sBins }; |
| } else if (nxt === CFG.segTok && cBins && sBins) { |
| pushDet(cBins, sBins); |
| entry.obj = true; |
| cBins = sBins = null; |
| if (dets.length >= 100) return; |
| } |
| emission.push(entry); |
| let useGather = true; |
| if (embNext) { eng.writeX(embNext); useGather = false; } |
| else if (st === 0) eng.device.queue.writeBuffer(eng.b.tokId, 0, new Uint32Array([nxt])); |
| const ts0 = performance.now(); |
| nxt = await eng.decodeStepAuto(useGather); |
| tSteps += performance.now() - ts0; nSteps++; |
| } |
| }; |
|
|
| |
| |
| |
| const clampBin = (v) => Math.max(0, Math.min(CFG.L - 1, Math.round(v))); |
| const buildDraft = () => { |
| if (!lastEmission || lastEmission.query !== query || lastEmission.terminal == null) return null; |
| const { entries, terminal: prevTerm, objMap } = lastEmission; |
| if (!entries.length) return null; |
| |
| |
| |
| const centerBins = (b) => [clampBin((b.x1 + b.x2) / 2 * (CFG.L - 1)), |
| clampBin((b.y1 + b.y2) / 2 * (CFG.L - 1))]; |
| const shifts = tracker ? tracker.boxes.map((t) => [ |
| (t.box.x1 + t.box.x2 - t.det.box.x1 - t.det.box.x2) / 2 * (CFG.L - 1), |
| (t.box.y1 + t.box.y2 - t.det.box.y1 - t.det.box.y2) / 2 * (CFG.L - 1)]) : []; |
| const med = (a) => (a.length ? a.slice().sort((x, y) => x - y)[a.length >> 1] : 0); |
| const mShift = [med(shifts.map((s) => s[0])), med(shifts.map((s) => s[1]))]; |
| let obj = 0; |
| const out = entries.map((e) => { |
| if (e.kind === 'c') { |
| const t = tracker && tracker.boxes[objMap[obj]]; |
| const raw = t ? centerBins(t.box) |
| : [clampBin(e.bins[0] + mShift[0]), clampBin(e.bins[1] + mShift[1])]; |
| const bins = [clampBin(raw[0] + specBias[0]), clampBin(raw[1] + specBias[1])]; |
| return { id: e.id, kind: 'c', bins, |
| emb: eng.encodeCoords('coord', coordFromBins(...bins)) }; |
| } |
| if (e.kind === 's') { |
| return { id: e.id, kind: 's', bins: e.bins.slice(), |
| emb: eng.encodeCoords('size', sizeFromBins(...e.bins)) }; |
| } |
| if (e.obj) obj++; |
| return { id: e.id, kind: 'g' }; |
| }); |
| out.push({ id: prevTerm, kind: 'g' }); |
| return out; |
| }; |
|
|
| |
| |
| |
| |
| |
| const binDiffs = []; |
| const runSpec = async (draft) => { |
| emission.push({ id: draft[0].id, kind: 'g' }); |
| let cBins = null, sBins = null, pos = 0, lastRows = null, repairs = 0; |
| |
| |
| |
| |
| const tryRepair = (fromIdx, bins) => { |
| let best = null, bestD = 65; |
| for (let j = fromIdx; j < draft.length - 1; j++) { |
| const e = draft[j]; |
| if (e.kind !== 'c') continue; |
| if (!draft[j + 1] || draft[j + 1].kind !== 's' || |
| !draft[j + 2] || draft[j + 2].id !== CFG.segTok) continue; |
| const d = Math.max(Math.abs(e.bins[0] - bins[0]), Math.abs(e.bins[1] - bins[1])); |
| if (d < bestD) { bestD = d; best = j; } |
| } |
| if (best == null) return null; |
| const candBins = draft[best].bins; |
| const tail = [ |
| { id: CFG.coordTok, kind: 'c', bins, |
| emb: eng.encodeCoords('coord', coordFromBins(...bins)) }, |
| draft[best + 1], draft[best + 2], |
| ...draft.slice(fromIdx, best), ...draft.slice(best + 3), |
| ]; |
| draft.length = fromIdx; |
| draft.push(...tail); |
| return candBins; |
| }; |
| while (pos < draft.length - 1) { |
| const k = Math.min(eng.maxK, draft.length - 1 - pos, MAX_T - 1 - eng.T); |
| if (k <= 0) { |
| emission.pop(); |
| return { nxt: draft[pos].id, preHead: lastRows }; |
| } |
| const items = draft.slice(pos, pos + k).map((e) => ({ id: e.id, emb: e.emb || null })); |
| const tv0 = performance.now(); |
| const r = await eng.verifyStep(items); |
| tSpec += performance.now() - tv0; nSpecRun++; |
| let repaired = false; |
| for (let i = 0; i < k; i++) { |
| const m = r.tokens[i]; |
| const exp = draft[pos + i + 1]; |
| const rows = () => ({ |
| coord: r.coordLogits.slice(i * 2048, (i + 1) * 2048), |
| size: r.sizeLogits.slice(i * 2048, (i + 1) * 2048) }); |
| if (STOP.has(m) || m === CFG.absence) { |
| eng.T = r.T0 + i + 1; terminal = m; nSpecOk += i; |
| return { done: true }; |
| } |
| if (m !== exp.id) { |
| eng.T = r.T0 + i + 1; nSpecOk += i; |
| return { nxt: m, preHead: rows() }; |
| } |
| if (exp.kind === 'c') { |
| const bins = coordBinsDedup(rows().coord); |
| if (Math.abs(bins[0] - exp.bins[0]) > OPTS.specTol || |
| Math.abs(bins[1] - exp.bins[1]) > OPTS.specTol) { |
| const candBins = repairs < 2 ? tryRepair(pos + i + 1, bins) : null; |
| if (candBins) { |
| binDiffs.push([bins[0] - candBins[0], bins[1] - candBins[1]]); |
| repairs++; nSpecOk += i + 1; |
| cBins = bins; sBins = null; |
| emission.push({ id: CFG.coordTok, kind: 'c', bins }); |
| eng.T = r.T0 + i + 1; |
| pos = pos + i + 1; |
| repaired = true; |
| break; |
| } |
| eng.T = r.T0 + i + 1; nSpecOk += i; |
| return { nxt: m, preHead: rows() }; |
| } |
| binDiffs.push([bins[0] - exp.bins[0], bins[1] - exp.bins[1]]); |
| cBins = bins; |
| emission.push({ id: exp.id, kind: 'c', bins }); |
| } else if (exp.kind === 's') { |
| const hl = rows().size; |
| const bins = [argmax(hl, 0, CFG.L), argmax(hl, CFG.L, CFG.L)]; |
| if (Math.abs(bins[0] - exp.bins[0]) > OPTS.specTol || |
| Math.abs(bins[1] - exp.bins[1]) > OPTS.specTol) { |
| if (repairs < 2) { |
| repairs++; nSpecOk += i + 1; |
| const tail = [{ id: CFG.sizeTok, kind: 's', bins, |
| emb: eng.encodeCoords('size', sizeFromBins(...bins)) }, |
| ...draft.slice(pos + i + 2)]; |
| draft.length = pos + i + 1; |
| draft.push(...tail); |
| sBins = bins; |
| emission.push({ id: CFG.sizeTok, kind: 's', bins }); |
| eng.T = r.T0 + i + 1; |
| pos = pos + i + 1; |
| repaired = true; |
| break; |
| } |
| eng.T = r.T0 + i + 1; nSpecOk += i; |
| return { nxt: m, preHead: rows() }; |
| } |
| sBins = bins; |
| emission.push({ id: exp.id, kind: 's', bins }); |
| } else { |
| const entry = { id: exp.id, kind: 'g' }; |
| if (exp.id === CFG.segTok && cBins && sBins) { |
| pushDet(cBins, sBins); |
| entry.obj = true; |
| cBins = sBins = null; |
| if (dets.length >= 100) { eng.T = r.T0 + i + 1; return { done: true }; } |
| } |
| emission.push(entry); |
| } |
| if (i === k - 1) lastRows = rows(); |
| } |
| if (repaired) continue; |
| eng.T = r.T0 + k; nSpecOk += k; pos += k; |
| } |
| return { done: true }; |
| }; |
|
|
| let nxt = argmax(logits), preHead = null, specDone = false; |
| if (OPTS.spec && lastEmission) { |
| const recent = specHist.slice(-4); |
| const hopeless = recent.length >= 4 && !recent.some(Boolean); |
| if (!hopeless || ++specSkips % 5 === 0) { |
| const draft = buildDraft(); |
| |
| if (draft && draft.length > 1 && !draft[0].emb && draft[0].id === nxt) { |
| const fed0 = draft.length - 1; |
| const rs = await runSpec(draft); |
| if (rs.done) specDone = true; |
| else { nxt = rs.nxt; preHead = rs.preHead; } |
| specHist.push(nSpecOk >= fed0 * 0.5 ? 1 : 0); |
| if (specHist.length > 16) specHist.shift(); |
| } |
| } |
| } |
| if (!specDone) await seqDecode(nxt, preHead); |
|
|
| |
| if (binDiffs.length) { |
| const med = (a) => a.slice().sort((x, y) => x - y)[a.length >> 1]; |
| const cl = (v) => Math.max(-200, Math.min(200, v)); |
| specBias = [cl(Math.round(specBias[0] / 2 + med(binDiffs.map((d) => d[0])) / 2)), |
| cl(Math.round(specBias[1] / 2 + med(binDiffs.map((d) => d[1])) / 2))]; |
| } |
|
|
| const final = nms(dets, 0.6); |
| lastEmission = { query, entries: emission, terminal, |
| objMap: dets.map((d) => final.indexOf(d)) }; |
| |
| |
| const shifted = final.map((d) => ({ |
| x1: d.box.x1 + panDX, y1: d.box.y1 + panDY, |
| x2: d.box.x2 + panDX, y2: d.box.y2 + panDY })); |
| tracker.setKeyframe(final, gray, fw, fh, shifted); |
| |
| lastKey = { gray, w: fw, h: fh, time: performance.now(), query, |
| boxes: final.map((d) => ({ ...d.box })) }; |
| const ms = performance.now() - t0; |
| console.log(`[engine] done ${ms.toFixed(0)}ms, ${final.length} detections (${dets.length} raw); ` + |
| `prefill=${tPrefill.toFixed(0)} steps=${tSteps.toFixed(0)}(${nSteps}x${(tSteps / Math.max(1, nSteps)).toFixed(1)}) heads=${tHeads.toFixed(0)}` + |
| (nSpecRun ? ` spec=${tSpec.toFixed(0)}(${nSpecRun} verify, ${nSpecOk} принято)` : '')); |
| reply(final, ms, { ...presStats, prefillMs: Math.round(tPrefill), |
| decodeMs: Math.round(tSteps + tSpec + tHeads), |
| seqSteps: nSteps, specRuns: nSpecRun, specAccepted: nSpecOk }); |
| } catch (e) { |
| console.error('[engine] detect error:', e); |
| post({ type: 'error', message: e.message || String(e) }); |
| } finally { |
| image.close?.(); |
| busy = false; |
| runQueued(); |
| } |
| } |
| function runQueued() { const q = queued; queued = null; if (q) detect(q); } |
|
|
| self.onmessage = async (e) => { |
| if (e.data.type === 'load') await load(e.data); |
| else if (e.data.type === 'detect') detect(e.data); |
| else if (e.data.type === 'config') { |
| const { type, ...opts } = e.data; |
| Object.assign(OPTS, opts); |
| |
| OPTS.videoRes = Math.max(128, Math.min(512, Math.round(OPTS.videoRes / 16) * 16)); |
| console.log('[engine] config:', JSON.stringify(OPTS)); |
| } |
| }; |
|
|