Spaces:
Running
Running
v0.3.1: lower peak memory while loading the model
Browse filesWeights are streamed into Cache Storage and read from disk tensor by tensor
during the GPU upload; phones load the model before booting the VM.
Source: github.com/erinoooo/LocalWebAgent@c0f9d66
- js/config.js +1 -1
- js/lazy-weights.js +83 -0
- js/llm-engine.js +4 -2
- js/llm-worker.js +16 -5
- js/main.js +19 -4
- sw.js +2 -2
js/config.js
CHANGED
|
@@ -1,5 +1,5 @@
|
|
| 1 |
// Central configuration. Everything that pins a version or a location lives here.
|
| 2 |
-
export const APP_VERSION = "0.3.
|
| 3 |
|
| 4 |
// --- Language model (loaded from the Hugging Face Hub, cached by transformers.js) ---
|
| 5 |
export const MODEL_ID = "LiquidAI/LFM3-1.7B-ONNX";
|
|
|
|
| 1 |
// Central configuration. Everything that pins a version or a location lives here.
|
| 2 |
+
export const APP_VERSION = "0.3.1";
|
| 3 |
|
| 4 |
// --- Language model (loaded from the Hugging Face Hub, cached by transformers.js) ---
|
| 5 |
export const MODEL_ID = "LiquidAI/LFM3-1.7B-ONNX";
|
js/lazy-weights.js
ADDED
|
@@ -0,0 +1,83 @@
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
| 1 |
+
// Low-memory loading of the model's external weights (model_<dtype>.onnx_data, ~1.4 GB).
|
| 2 |
+
//
|
| 3 |
+
// By default transformers.js reads the whole file into one Uint8Array and hands it to
|
| 4 |
+
// onnxruntime, which then copies it tensor by tensor into GPU buffers. Until that finishes
|
| 5 |
+
// the weights exist twice: once in JS memory and once on the GPU. On phones and other
|
| 6 |
+
// devices with shared CPU/GPU memory that peak is what gets the tab's GPU process killed.
|
| 7 |
+
//
|
| 8 |
+
// Instead we keep the file in Cache Storage (streamed there, never fully in memory) and
|
| 9 |
+
// give onnxruntime a Blob-backed stand-in. onnxruntime only ever calls .byteLength and
|
| 10 |
+
// .subarray(begin, end) on mounted external data, so each tensor is read from disk with
|
| 11 |
+
// FileReaderSync right before it is uploaded. Peak extra memory becomes one tensor.
|
| 12 |
+
//
|
| 13 |
+
// Worker-only (FileReaderSync). Callers fall back to the default path if this throws.
|
| 14 |
+
|
| 15 |
+
const CACHE_NAME = "transformers-cache"; // the cache transformers.js uses; setup.js checks it too
|
| 16 |
+
|
| 17 |
+
/** A Uint8Array (so transformers.js passes it through untouched) whose bytes live in a Blob. */
|
| 18 |
+
class BlobBytes extends Uint8Array {
|
| 19 |
+
#blob;
|
| 20 |
+
constructor(blob) {
|
| 21 |
+
super(0);
|
| 22 |
+
this.#blob = blob;
|
| 23 |
+
}
|
| 24 |
+
get byteLength() { return this.#blob.size; }
|
| 25 |
+
get length() { return this.#blob.size; }
|
| 26 |
+
subarray(begin = 0, end = this.#blob.size) {
|
| 27 |
+
return new Uint8Array(new FileReaderSync().readAsArrayBuffer(this.#blob.slice(begin, end)));
|
| 28 |
+
}
|
| 29 |
+
}
|
| 30 |
+
|
| 31 |
+
/** Streams url into Cache Storage, reporting progress like transformers.js does. */
|
| 32 |
+
async function download(cache, url, file, progress) {
|
| 33 |
+
const res = await fetch(url);
|
| 34 |
+
if (!res.ok || !res.body) throw new Error(`Download failed (${res.status}) for ${file}`);
|
| 35 |
+
const total = Number(res.headers.get("Content-Length")) || 0;
|
| 36 |
+
let loaded = 0;
|
| 37 |
+
let last = 0;
|
| 38 |
+
progress?.({ status: "initiate", file, loaded: 0, total });
|
| 39 |
+
const counted = res.body.pipeThrough(new TransformStream({
|
| 40 |
+
transform(chunk, ctl) {
|
| 41 |
+
loaded += chunk.byteLength;
|
| 42 |
+
const now = performance.now();
|
| 43 |
+
if (now - last > 200) { last = now; progress?.({ status: "progress", file, loaded, total }); }
|
| 44 |
+
ctl.enqueue(chunk);
|
| 45 |
+
},
|
| 46 |
+
}));
|
| 47 |
+
const headers = new Headers(res.headers);
|
| 48 |
+
await cache.put(url, new Response(counted, { status: 200, headers }));
|
| 49 |
+
if (total && loaded !== total) {
|
| 50 |
+
await cache.delete(url);
|
| 51 |
+
throw new Error(`Incomplete download of ${file} (${loaded} of ${total} bytes)`);
|
| 52 |
+
}
|
| 53 |
+
progress?.({ status: "done", file, loaded, total: total || loaded });
|
| 54 |
+
}
|
| 55 |
+
|
| 56 |
+
/**
|
| 57 |
+
* Returns from_pretrained options that make onnxruntime read the external weights
|
| 58 |
+
* lazily from Cache Storage, downloading them first if needed.
|
| 59 |
+
*/
|
| 60 |
+
export async function lazyWeightsOptions(modelId, dtype, progress) {
|
| 61 |
+
if (typeof FileReaderSync === "undefined" || !self.caches) throw new Error("lazy weights unsupported here");
|
| 62 |
+
const name = `model_${dtype}.onnx_data`;
|
| 63 |
+
const file = `onnx/${name}`;
|
| 64 |
+
const url = `https://huggingface.co/${modelId}/resolve/main/${file}`;
|
| 65 |
+
const cache = await caches.open(CACHE_NAME);
|
| 66 |
+
let res = await cache.match(url);
|
| 67 |
+
if (!res) {
|
| 68 |
+
await download(cache, url, file, progress);
|
| 69 |
+
res = await cache.match(url);
|
| 70 |
+
if (!res) throw new Error("weights were not stored in Cache Storage");
|
| 71 |
+
} else {
|
| 72 |
+
const size = Number(res.headers.get("Content-Length")) || 0;
|
| 73 |
+
progress?.({ status: "done", file, loaded: size, total: size });
|
| 74 |
+
}
|
| 75 |
+
const blob = await res.blob(); // disk-backed in Chromium; not read into memory
|
| 76 |
+
if (!blob.size) throw new Error("cached weights are empty");
|
| 77 |
+
return {
|
| 78 |
+
// Stop transformers.js from fetching the .onnx_data itself...
|
| 79 |
+
use_external_data_format: false,
|
| 80 |
+
// ...and mount ours under the name the ONNX graph refers to.
|
| 81 |
+
session_options: { externalData: [{ path: name, data: new BlobBytes(blob) }] },
|
| 82 |
+
};
|
| 83 |
+
}
|
js/llm-engine.js
CHANGED
|
@@ -64,10 +64,12 @@ export function createEngine(tf) {
|
|
| 64 |
const int64 = (arr, dims) => new Tensor("int64", BigInt64Array.from(arr, BigInt), dims);
|
| 65 |
const ones = (n) => new Tensor("int64", new BigInt64Array(n).fill(1n), [1, n]);
|
| 66 |
|
| 67 |
-
|
|
|
|
|
|
|
| 68 |
tokenizer = await AutoTokenizer.from_pretrained(modelId, { progress_callback });
|
| 69 |
onStatus?.("Downloading / loading model weights");
|
| 70 |
-
model = await AutoModelForCausalLM.from_pretrained(modelId, { dtype, device, progress_callback });
|
| 71 |
const te = tokenizer.encode("</think>", { add_special_tokens: false });
|
| 72 |
thinkEndId = te.length === 1 ? te[0] : -1;
|
| 73 |
// The LFM3 ONNX graph takes `logits_to_keep` (rank-1 int64). transformers.js only
|
|
|
|
| 64 |
const int64 = (arr, dims) => new Tensor("int64", BigInt64Array.from(arr, BigInt), dims);
|
| 65 |
const ones = (n) => new Tensor("int64", new BigInt64Array(n).fill(1n), [1, n]);
|
| 66 |
|
| 67 |
+
// modelOptions: extra from_pretrained options (the browser worker uses it to hand
|
| 68 |
+
// onnxruntime the weights as a lazily read, disk-backed buffer).
|
| 69 |
+
async function load(modelId, { dtype, device, progress_callback, onStatus, modelOptions } = {}) {
|
| 70 |
tokenizer = await AutoTokenizer.from_pretrained(modelId, { progress_callback });
|
| 71 |
onStatus?.("Downloading / loading model weights");
|
| 72 |
+
model = await AutoModelForCausalLM.from_pretrained(modelId, { dtype, device, progress_callback, ...modelOptions });
|
| 73 |
const te = tokenizer.encode("</think>", { add_special_tokens: false });
|
| 74 |
thinkEndId = te.length === 1 ? te[0] : -1;
|
| 75 |
// The LFM3 ONNX graph takes `logits_to_keep` (rank-1 int64). transformers.js only
|
js/llm-worker.js
CHANGED
|
@@ -2,6 +2,7 @@
|
|
| 2 |
// The generation logic lives in llm-engine.js (shared with the Node test harness).
|
| 3 |
import * as tf from "../vendor/transformers.min.js";
|
| 4 |
import { createEngine } from "./llm-engine.js";
|
|
|
|
| 5 |
|
| 6 |
tf.env.allowLocalModels = false;
|
| 7 |
tf.env.backends.onnx.wasm.wasmPaths = {
|
|
@@ -36,14 +37,24 @@ async function load({ modelId, dtype, device }) {
|
|
| 36 |
const info = adapter.info ?? {};
|
| 37 |
post({ type: "gpu", vendor: info.vendor, architecture: info.architecture, description: info.description });
|
| 38 |
}
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
| 39 |
await engine.load(modelId, {
|
| 40 |
dtype,
|
| 41 |
device,
|
| 42 |
-
progress_callback
|
| 43 |
-
|
| 44 |
-
post({ type: "progress", file: p.file, loaded: p.loaded ?? 0, total: p.total ?? 0, status: p.status });
|
| 45 |
-
}
|
| 46 |
-
},
|
| 47 |
onStatus: (text) => post({ type: "status", text }),
|
| 48 |
});
|
| 49 |
post({ type: "ready" });
|
|
|
|
| 2 |
// The generation logic lives in llm-engine.js (shared with the Node test harness).
|
| 3 |
import * as tf from "../vendor/transformers.min.js";
|
| 4 |
import { createEngine } from "./llm-engine.js";
|
| 5 |
+
import { lazyWeightsOptions } from "./lazy-weights.js";
|
| 6 |
|
| 7 |
tf.env.allowLocalModels = false;
|
| 8 |
tf.env.backends.onnx.wasm.wasmPaths = {
|
|
|
|
| 37 |
const info = adapter.info ?? {};
|
| 38 |
post({ type: "gpu", vendor: info.vendor, architecture: info.architecture, description: info.description });
|
| 39 |
}
|
| 40 |
+
const progress_callback = (p) => {
|
| 41 |
+
if (p.status === "progress" || p.status === "done" || p.status === "initiate") {
|
| 42 |
+
post({ type: "progress", file: p.file, loaded: p.loaded ?? 0, total: p.total ?? 0, status: p.status });
|
| 43 |
+
}
|
| 44 |
+
};
|
| 45 |
+
// Read the ~1.4 GB of weights from disk tensor by tensor instead of holding them all
|
| 46 |
+
// in memory while they are copied to the GPU (see lazy-weights.js).
|
| 47 |
+
let modelOptions;
|
| 48 |
+
try {
|
| 49 |
+
modelOptions = await lazyWeightsOptions(modelId, dtype, progress_callback);
|
| 50 |
+
} catch (err) {
|
| 51 |
+
console.warn("Low-memory weight loading unavailable, using the default path:", err);
|
| 52 |
+
}
|
| 53 |
await engine.load(modelId, {
|
| 54 |
dtype,
|
| 55 |
device,
|
| 56 |
+
progress_callback,
|
| 57 |
+
modelOptions,
|
|
|
|
|
|
|
|
|
|
| 58 |
onStatus: (text) => post({ type: "status", text }),
|
| 59 |
});
|
| 60 |
post({ type: "ready" });
|
js/main.js
CHANGED
|
@@ -208,7 +208,7 @@ async function start() {
|
|
| 208 |
agent = new Agent({ llm, vm, settings, ui: chatUI });
|
| 209 |
setComposerEnabled();
|
| 210 |
|
| 211 |
-
const
|
| 212 |
status("vm", "busy", "Booting");
|
| 213 |
try {
|
| 214 |
diskManifest ??= await setup.installedDisk();
|
|
@@ -223,15 +223,30 @@ async function start() {
|
|
| 223 |
vm.term?.writeln("\r\n\x1b[31mFailed to start Linux: " + err.message + "\x1b[0m");
|
| 224 |
}
|
| 225 |
setComposerEnabled();
|
| 226 |
-
}
|
| 227 |
-
const
|
| 228 |
console.error(err);
|
| 229 |
addNotice("The model failed to load: " + err.message, "error");
|
| 230 |
});
|
| 231 |
-
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
| 232 |
setComposerEnabled();
|
| 233 |
}
|
| 234 |
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
| 235 |
// ---------- conversation rendering ----------
|
| 236 |
const thread = () => $("#messages");
|
| 237 |
const scroller = () => $("#scroller");
|
|
|
|
| 208 |
agent = new Agent({ llm, vm, settings, ui: chatUI });
|
| 209 |
setComposerEnabled();
|
| 210 |
|
| 211 |
+
const bootVm = async () => {
|
| 212 |
status("vm", "busy", "Booting");
|
| 213 |
try {
|
| 214 |
diskManifest ??= await setup.installedDisk();
|
|
|
|
| 223 |
vm.term?.writeln("\r\n\x1b[31mFailed to start Linux: " + err.message + "\x1b[0m");
|
| 224 |
}
|
| 225 |
setComposerEnabled();
|
| 226 |
+
};
|
| 227 |
+
const loadModelOnce = () => llm.ready ? Promise.resolve() : loadModel().catch((err) => {
|
| 228 |
console.error(err);
|
| 229 |
addNotice("The model failed to load: " + err.message, "error");
|
| 230 |
});
|
| 231 |
+
if (memoryConstrained()) {
|
| 232 |
+
// Upload the weights to the GPU before Linux takes its share of memory: on phones
|
| 233 |
+
// the two together at peak are what makes the browser kill the GPU process.
|
| 234 |
+
status("vm", "busy", "Waiting for the model");
|
| 235 |
+
await loadModelOnce();
|
| 236 |
+
await bootVm();
|
| 237 |
+
} else {
|
| 238 |
+
await Promise.all([bootVm(), loadModelOnce()]);
|
| 239 |
+
}
|
| 240 |
setComposerEnabled();
|
| 241 |
}
|
| 242 |
|
| 243 |
+
// Phones, tablets and machines reporting under 8 GB of RAM (Chrome caps deviceMemory at 8).
|
| 244 |
+
function memoryConstrained() {
|
| 245 |
+
if (navigator.userAgentData?.mobile) return true;
|
| 246 |
+
if (/Android|iPhone|iPad|Mobile/i.test(navigator.userAgent)) return true;
|
| 247 |
+
return (navigator.deviceMemory ?? 8) < 8;
|
| 248 |
+
}
|
| 249 |
+
|
| 250 |
// ---------- conversation rendering ----------
|
| 251 |
const thread = () => $("#messages");
|
| 252 |
const scroller = () => $("#scroller");
|
sw.js
CHANGED
|
@@ -6,7 +6,7 @@
|
|
| 6 |
// (__vm__/disk.ext2), backed by the chunks the setup screen stored in Cache Storage.
|
| 7 |
// 4. Adds the cross-origin isolation headers (COOP/COEP) that SharedArrayBuffer,
|
| 8 |
// CheerpX and multi-threaded WASM need, even when a host forgets to send them.
|
| 9 |
-
const SHELL_VERSION = "lwa-shell-0.3.
|
| 10 |
const CX_ORIGIN = "https://cxrtnc.leaningtech.com";
|
| 11 |
const CX_CACHE_PREFIX = "lwa-cheerpx-";
|
| 12 |
const DISK_CACHE = "lwa-disk";
|
|
@@ -18,7 +18,7 @@ const chunkKey = (i) => SCOPE + "__vm__/chunk/" + i;
|
|
| 18 |
const SHELL_FILES = [
|
| 19 |
"./", "index.html", "css/app.css", "icon.svg",
|
| 20 |
"js/config.js", "js/config-defaults.js", "js/main.js", "js/setup.js", "js/vm.js", "js/agent.js", "js/llm.js",
|
| 21 |
-
"js/llm-worker.js", "js/llm-engine.js", "js/prompts.js", "js/toolcalls.js", "js/markdown.js", "js/files.js",
|
| 22 |
"vendor/transformers.min.js", "vendor/highlight.min.js",
|
| 23 |
"vendor/ort/ort-wasm-simd-threaded.asyncify.mjs", "vendor/ort/ort-wasm-simd-threaded.asyncify.wasm",
|
| 24 |
"vendor/xterm/xterm.mjs", "vendor/xterm/xterm.css", "vendor/xterm/addon-fit.mjs",
|
|
|
|
| 6 |
// (__vm__/disk.ext2), backed by the chunks the setup screen stored in Cache Storage.
|
| 7 |
// 4. Adds the cross-origin isolation headers (COOP/COEP) that SharedArrayBuffer,
|
| 8 |
// CheerpX and multi-threaded WASM need, even when a host forgets to send them.
|
| 9 |
+
const SHELL_VERSION = "lwa-shell-0.3.1";
|
| 10 |
const CX_ORIGIN = "https://cxrtnc.leaningtech.com";
|
| 11 |
const CX_CACHE_PREFIX = "lwa-cheerpx-";
|
| 12 |
const DISK_CACHE = "lwa-disk";
|
|
|
|
| 18 |
const SHELL_FILES = [
|
| 19 |
"./", "index.html", "css/app.css", "icon.svg",
|
| 20 |
"js/config.js", "js/config-defaults.js", "js/main.js", "js/setup.js", "js/vm.js", "js/agent.js", "js/llm.js",
|
| 21 |
+
"js/llm-worker.js", "js/llm-engine.js", "js/lazy-weights.js", "js/prompts.js", "js/toolcalls.js", "js/markdown.js", "js/files.js",
|
| 22 |
"vendor/transformers.min.js", "vendor/highlight.min.js",
|
| 23 |
"vendor/ort/ort-wasm-simd-threaded.asyncify.mjs", "vendor/ort/ort-wasm-simd-threaded.asyncify.wasm",
|
| 24 |
"vendor/xterm/xterm.mjs", "vendor/xterm/xterm.css", "vendor/xterm/addon-fit.mjs",
|