Spaces:
Running
Running
Download js/llm-worker.js from ereniko/LocalWebAgent: direct link, hf CLI and curl.
- Browser
- Download file 3.75 kB
-
https://huggingface.co/spaces/ereniko/LocalWebAgent/resolve/main/js/llm-worker.js
- Command line
-
hf download hf://spaces/ereniko/LocalWebAgent/js/llm-worker.js
-
curl -L -o llm-worker.js https://huggingface.co/spaces/ereniko/LocalWebAgent/resolve/main/js/llm-worker.js
3.75 kB
| // Web Worker that owns LFM3-1.7B, running on WebGPU via transformers.js + onnxruntime-web. | |
| // The generation logic lives in llm-engine.js (shared with the Node test harness). | |
| import * as tf from "../vendor/transformers.min.js"; | |
| import { createEngine } from "./llm-engine.js"; | |
| import { lazyWeightsOptions, cachedRepoText } from "./lazy-weights.js"; | |
| tf.env.allowLocalModels = false; | |
| tf.env.backends.onnx.wasm.wasmPaths = { | |
| mjs: new URL("../vendor/ort/ort-wasm-simd-threaded.asyncify.mjs", import.meta.url).href, | |
| wasm: new URL("../vendor/ort/ort-wasm-simd-threaded.asyncify.wasm", import.meta.url).href, | |
| }; | |
| const engine = createEngine(tf); | |
| const post = (msg) => self.postMessage(msg); | |
| self.onmessage = async ({ data }) => { | |
| try { | |
| if (data.type === "load") await load(data); | |
| else if (data.type === "generate") { | |
| const t0 = performance.now(); | |
| const result = await engine.generate(data.messages, data.tools, data.options, (ev) => post({ ...ev, id: data.id })); | |
| post({ type: "done", id: data.id, ...result, totalMs: performance.now() - t0 }); | |
| } else if (data.type === "stop") engine.stop(); | |
| else if (data.type === "reset") await engine.reset(); | |
| } catch (err) { | |
| post({ type: "error", id: data.id, message: String(err?.message ?? err), stack: err?.stack }); | |
| } | |
| }; | |
| async function load({ modelId, dtype, graphs = [{ file: "model", chunks: 1 }], textConfig = false, chatTemplateFile = false, emptyInputs = null, fullPrefill = false, device }) { | |
| if (device === "webgpu") { | |
| const adapter = await navigator.gpu?.requestAdapter({ powerPreference: "high-performance" }); | |
| if (!adapter) throw new Error("WebGPU is not available in this browser. Use a recent desktop Chrome or Edge."); | |
| if (dtype.includes("f16") && !adapter.features.has("shader-f16")) { | |
| throw new Error("This GPU/browser lacks WebGPU 'shader-f16' support, which the q4f16 model needs."); | |
| } | |
| const info = adapter.info ?? {}; | |
| post({ type: "gpu", vendor: info.vendor, architecture: info.architecture, description: info.description }); | |
| } | |
| const progress_callback = (p) => { | |
| if (p.status === "progress" || p.status === "done" || p.status === "initiate") { | |
| post({ type: "progress", file: p.file, loaded: p.loaded ?? 0, total: p.total ?? 0, status: p.status }); | |
| } | |
| }; | |
| // Read external weights from disk tensor by tensor instead of holding them all in | |
| // memory while they are copied to the GPU (see lazy-weights.js). Models whose weights | |
| // are inside the .onnx file itself (chunks = 0) load the default way. | |
| let modelOptions = {}; | |
| try { | |
| if (graphs.some((g) => g.chunks > 0)) modelOptions = await lazyWeightsOptions(modelId, dtype, graphs, progress_callback); | |
| } catch (err) { | |
| console.warn("Low-memory weight loading unavailable, using the default path:", err); | |
| } | |
| // Text-only use of a multimodal repo: its graph has another name, its config.json describes | |
| // the whole multimodal model (use the text part), and its chat template is a separate file. | |
| if (graphs.length === 1 && graphs[0].file !== "model") modelOptions.model_file_name = graphs[0].file; | |
| if (textConfig) { | |
| const full = JSON.parse(await cachedRepoText(modelId, "config.json")); | |
| modelOptions.config = { | |
| ...full.text_config, | |
| "transformers.js_config": { kv_cache_dtype: { q4f16: "float16", fp16: "float16" }, ...full["transformers.js_config"] }, | |
| }; | |
| } | |
| const chatTemplate = chatTemplateFile ? await cachedRepoText(modelId, "chat_template.jinja") : undefined; | |
| await engine.load(modelId, { | |
| dtype, | |
| device, | |
| progress_callback, | |
| modelOptions, | |
| chatTemplate, | |
| emptyInputs, | |
| fullPrefill, | |
| onStatus: (text) => post({ type: "status", text }), | |
| }); | |
| post({ type: "ready" }); | |
| } | |