// Web Worker that owns LFM3-1.7B, running on WebGPU via transformers.js + onnxruntime-web. // The generation logic lives in llm-engine.js (shared with the Node test harness). import * as tf from "../vendor/transformers.min.js"; import { createEngine } from "./llm-engine.js"; import { lazyWeightsOptions, cachedRepoText } from "./lazy-weights.js"; tf.env.allowLocalModels = false; tf.env.backends.onnx.wasm.wasmPaths = { mjs: new URL("../vendor/ort/ort-wasm-simd-threaded.asyncify.mjs", import.meta.url).href, wasm: new URL("../vendor/ort/ort-wasm-simd-threaded.asyncify.wasm", import.meta.url).href, }; const engine = createEngine(tf); const post = (msg) => self.postMessage(msg); self.onmessage = async ({ data }) => { try { if (data.type === "load") await load(data); else if (data.type === "generate") { const t0 = performance.now(); const result = await engine.generate(data.messages, data.tools, data.options, (ev) => post({ ...ev, id: data.id })); post({ type: "done", id: data.id, ...result, totalMs: performance.now() - t0 }); } else if (data.type === "stop") engine.stop(); else if (data.type === "reset") await engine.reset(); } catch (err) { post({ type: "error", id: data.id, message: String(err?.message ?? err), stack: err?.stack }); } }; async function load({ modelId, dtype, graphs = [{ file: "model", chunks: 1 }], textConfig = false, chatTemplateFile = false, emptyInputs = null, fullPrefill = false, device }) { if (device === "webgpu") { const adapter = await navigator.gpu?.requestAdapter({ powerPreference: "high-performance" }); if (!adapter) throw new Error("WebGPU is not available in this browser. Use a recent desktop Chrome or Edge."); if (dtype.includes("f16") && !adapter.features.has("shader-f16")) { throw new Error("This GPU/browser lacks WebGPU 'shader-f16' support, which the q4f16 model needs."); } const info = adapter.info ?? {}; post({ type: "gpu", vendor: info.vendor, architecture: info.architecture, description: info.description }); } const progress_callback = (p) => { if (p.status === "progress" || p.status === "done" || p.status === "initiate") { post({ type: "progress", file: p.file, loaded: p.loaded ?? 0, total: p.total ?? 0, status: p.status }); } }; // Read external weights from disk tensor by tensor instead of holding them all in // memory while they are copied to the GPU (see lazy-weights.js). Models whose weights // are inside the .onnx file itself (chunks = 0) load the default way. let modelOptions = {}; try { if (graphs.some((g) => g.chunks > 0)) modelOptions = await lazyWeightsOptions(modelId, dtype, graphs, progress_callback); } catch (err) { console.warn("Low-memory weight loading unavailable, using the default path:", err); } // Text-only use of a multimodal repo: its graph has another name, its config.json describes // the whole multimodal model (use the text part), and its chat template is a separate file. if (graphs.length === 1 && graphs[0].file !== "model") modelOptions.model_file_name = graphs[0].file; if (textConfig) { const full = JSON.parse(await cachedRepoText(modelId, "config.json")); modelOptions.config = { ...full.text_config, "transformers.js_config": { kv_cache_dtype: { q4f16: "float16", fp16: "float16" }, ...full["transformers.js_config"] }, }; } const chatTemplate = chatTemplateFile ? await cachedRepoText(modelId, "chat_template.jinja") : undefined; await engine.load(modelId, { dtype, device, progress_callback, modelOptions, chatTemplate, emptyInputs, fullPrefill, onStatus: (text) => post({ type: "status", text }), }); post({ type: "ready" }); }