LocalWebAgent / js /llm-worker.js
ereniko's picture
v0.7.0: Qwen3.5 2B/4B, SmolLM3 3B, Llama 3.2 3B, Jan-nano 4B; drop Qwen3 4B
6b1f716 unverified
Raw History Blame Contribute Delete
3.75 kB
// Web Worker that owns LFM3-1.7B, running on WebGPU via transformers.js + onnxruntime-web.
// The generation logic lives in llm-engine.js (shared with the Node test harness).
import * as tf from "../vendor/transformers.min.js";
import { createEngine } from "./llm-engine.js";
import { lazyWeightsOptions, cachedRepoText } from "./lazy-weights.js";
tf.env.allowLocalModels = false;
tf.env.backends.onnx.wasm.wasmPaths = {
mjs: new URL("../vendor/ort/ort-wasm-simd-threaded.asyncify.mjs", import.meta.url).href,
wasm: new URL("../vendor/ort/ort-wasm-simd-threaded.asyncify.wasm", import.meta.url).href,
};
const engine = createEngine(tf);
const post = (msg) => self.postMessage(msg);
self.onmessage = async ({ data }) => {
try {
if (data.type === "load") await load(data);
else if (data.type === "generate") {
const t0 = performance.now();
const result = await engine.generate(data.messages, data.tools, data.options, (ev) => post({ ...ev, id: data.id }));
post({ type: "done", id: data.id, ...result, totalMs: performance.now() - t0 });
} else if (data.type === "stop") engine.stop();
else if (data.type === "reset") await engine.reset();
} catch (err) {
post({ type: "error", id: data.id, message: String(err?.message ?? err), stack: err?.stack });
}
};
async function load({ modelId, dtype, graphs = [{ file: "model", chunks: 1 }], textConfig = false, chatTemplateFile = false, emptyInputs = null, fullPrefill = false, device }) {
if (device === "webgpu") {
const adapter = await navigator.gpu?.requestAdapter({ powerPreference: "high-performance" });
if (!adapter) throw new Error("WebGPU is not available in this browser. Use a recent desktop Chrome or Edge.");
if (dtype.includes("f16") && !adapter.features.has("shader-f16")) {
throw new Error("This GPU/browser lacks WebGPU 'shader-f16' support, which the q4f16 model needs.");
}
const info = adapter.info ?? {};
post({ type: "gpu", vendor: info.vendor, architecture: info.architecture, description: info.description });
}
const progress_callback = (p) => {
if (p.status === "progress" || p.status === "done" || p.status === "initiate") {
post({ type: "progress", file: p.file, loaded: p.loaded ?? 0, total: p.total ?? 0, status: p.status });
}
};
// Read external weights from disk tensor by tensor instead of holding them all in
// memory while they are copied to the GPU (see lazy-weights.js). Models whose weights
// are inside the .onnx file itself (chunks = 0) load the default way.
let modelOptions = {};
try {
if (graphs.some((g) => g.chunks > 0)) modelOptions = await lazyWeightsOptions(modelId, dtype, graphs, progress_callback);
} catch (err) {
console.warn("Low-memory weight loading unavailable, using the default path:", err);
}
// Text-only use of a multimodal repo: its graph has another name, its config.json describes
// the whole multimodal model (use the text part), and its chat template is a separate file.
if (graphs.length === 1 && graphs[0].file !== "model") modelOptions.model_file_name = graphs[0].file;
if (textConfig) {
const full = JSON.parse(await cachedRepoText(modelId, "config.json"));
modelOptions.config = {
...full.text_config,
"transformers.js_config": { kv_cache_dtype: { q4f16: "float16", fp16: "float16" }, ...full["transformers.js_config"] },
};
}
const chatTemplate = chatTemplateFile ? await cachedRepoText(modelId, "chat_template.jinja") : undefined;
await engine.load(modelId, {
dtype,
device,
progress_callback,
modelOptions,
chatTemplate,
emptyInputs,
fullPrefill,
onStatus: (text) => post({ type: "status", text }),
});
post({ type: "ready" });
}